2024-09-13T13:59:52.045 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T13:59:52.070 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T13:59:52.197 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551 branch: wip-aclamk-cbt-combined description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: akupczyk@redhat.com first_in_suite: false job_id: '7903551' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-aclamk-cbt-combined ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-aclamk-cbt-combined sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 owner: scheduled_akupczyk@teuthology priority: 160 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 6545 sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 sleep_before_teardown: 0 subset: 682/13280 suite: upgrade suite_branch: wip-aclamk-cbt-combined suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 targets: smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF0ApP33P4D+m1iqXZFLZK/fLHR9nldTUvxVX8J5co8N2pM4FAqDVTsSImkiewzS8Pcfa1n48bn4SiN4pJ6U61M= smithi145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMbM5AcfthDE6XALNjOPuAObDw3AeuucAbaR0p6bvi1e7YWxdShLf9Z/lf8bgy99XhRLcyLToegJscTUoluOcts= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_10:00:18 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: akupczyk verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-13T13:59:52.197 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa; will attempt to use it 2024-09-13T13:59:52.198 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks 2024-09-13T13:59:52.198 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T13:59:52.201 INFO:teuthology.task.internal:Checking packages... 2024-09-13T13:59:52.226 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd58f73069a8157a9ace12a0247ee3e8497707ea9' 2024-09-13T13:59:52.226 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T13:59:52.226 INFO:teuthology.packaging:ref: None 2024-09-13T13:59:52.227 INFO:teuthology.packaging:tag: None 2024-09-13T13:59:52.227 INFO:teuthology.packaging:branch: wip-aclamk-cbt-combined 2024-09-13T13:59:52.227 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T13:59:52.227 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-aclamk-cbt-combined 2024-09-13T13:59:52.386 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4697.gd58f7306 2024-09-13T13:59:52.387 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T13:59:52.397 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T13:59:52.397 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T13:59:52.445 INFO:teuthology.task.internal:Saving configuration 2024-09-13T13:59:52.464 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T13:59:52.474 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T13:59:52.507 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:54:53.675927', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF0ApP33P4D+m1iqXZFLZK/fLHR9nldTUvxVX8J5co8N2pM4FAqDVTsSImkiewzS8Pcfa1n48bn4SiN4pJ6U61M='} 2024-09-13T13:59:52.540 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:54:53.674203', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMbM5AcfthDE6XALNjOPuAObDw3AeuucAbaR0p6bvi1e7YWxdShLf9Z/lf8bgy99XhRLcyLToegJscTUoluOcts='} 2024-09-13T13:59:52.540 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T13:59:52.552 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T13:59:52.552 INFO:teuthology.task.internal:roles: ubuntu@smithi145.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T13:59:52.552 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T13:59:52.648 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8a10cd7be0>, signals=[15]) 2024-09-13T13:59:52.648 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T13:59:52.659 INFO:teuthology.task.internal:Opening connections... 2024-09-13T13:59:52.660 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com 2024-09-13T13:59:52.661 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:52.765 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi145.front.sepia.ceph.com 2024-09-13T13:59:52.766 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:52.853 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T13:59:52.865 DEBUG:teuthology.orchestra.run.smithi026:> uname -m 2024-09-13T13:59:52.886 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64 2024-09-13T13:59:52.886 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="CentOS Stream" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="9" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:ID="centos" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="rhel fedora" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="9" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el9" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T13:59:52.945 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:LOGO="fedora-logo-icon" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://centos.org/" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T13:59:52.946 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T13:59:52.947 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server 2024-09-13T13:59:52.979 DEBUG:teuthology.orchestra.run.smithi145:> uname -m 2024-09-13T13:59:52.998 INFO:teuthology.orchestra.run.smithi145.stdout:x86_64 2024-09-13T13:59:52.999 DEBUG:teuthology.orchestra.run.smithi145:> cat /etc/os-release 2024-09-13T13:59:53.057 INFO:teuthology.orchestra.run.smithi145.stdout:NAME="CentOS Stream" 2024-09-13T13:59:53.057 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION="9" 2024-09-13T13:59:53.057 INFO:teuthology.orchestra.run.smithi145.stdout:ID="centos" 2024-09-13T13:59:53.057 INFO:teuthology.orchestra.run.smithi145.stdout:ID_LIKE="rhel fedora" 2024-09-13T13:59:53.057 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_ID="9" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:PLATFORM_ID="platform:el9" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:ANSI_COLOR="0;31" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:LOGO="fedora-logo-icon" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:HOME_URL="https://centos.org/" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T13:59:53.058 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T13:59:53.059 INFO:teuthology.lock.ops:Updating smithi145.front.sepia.ceph.com on lock server 2024-09-13T13:59:53.088 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T13:59:53.097 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T13:59:53.122 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T13:59:53.122 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:59:53.124 DEBUG:teuthology.orchestra.run.smithi145:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:59:53.142 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T13:59:53.152 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T13:59:53.153 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:59:53.182 DEBUG:teuthology.orchestra.run.smithi145:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:59:53.286 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T13:59:53.354 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T13:59:53.372 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T13:59:53.372 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T13:59:53.372 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:59:53.373 DEBUG:teuthology.orchestra.run.smithi145:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:59:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:59:53.390 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2024-09-13T13:59:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:59:53.391 DEBUG:teuthology.orchestra.run.smithi145:> uname -r 2024-09-13T13:59:53.449 INFO:teuthology.orchestra.run.smithi026.stdout:5.14.0-503.el9.x86_64 2024-09-13T13:59:53.449 INFO:teuthology.task.kernel:Running kernel on smithi026: 5.14.0-503.el9.x86_64 2024-09-13T13:59:53.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel 2024-09-13T13:59:53.450 INFO:teuthology.orchestra.run.smithi145.stdout:5.14.0-503.el9.x86_64 2024-09-13T13:59:53.451 INFO:teuthology.task.kernel:Running kernel on smithi145: 5.14.0-503.el9.x86_64 2024-09-13T13:59:53.451 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum install -y kernel 2024-09-13T13:59:56.023 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-09-13T13:59:57.092 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.2 MB 00:01 2024-09-13T14:00:09.847 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - AppStream 1.6 MB/s | 20 MB 00:12 2024-09-13T14:00:16.246 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - CRB 2.5 MB/s | 6.5 MB 00:02 2024-09-13T14:00:16.674 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 20 MB 00:18 2024-09-13T14:00:17.722 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 19 kB 00:00 2024-09-13T14:00:19.368 INFO:teuthology.orchestra.run.smithi145.stdout:Extra Packages for Enterprise Linux 15 MB/s | 23 MB 00:01 2024-09-13T14:00:21.092 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.5 MB 00:00 2024-09-13T14:00:22.676 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - Extras packages 56 kB/s | 19 kB 00:00 2024-09-13T14:00:23.855 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-13T14:00:24.815 INFO:teuthology.orchestra.run.smithi145.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-13T14:00:26.839 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:00:26.839 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:00:26.932 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-09-13T14:00:26.938 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:00:26.938 INFO:teuthology.orchestra.run.smithi145.stdout: Package Architecture Version Repository Size 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:Installing: 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:Installing dependencies: 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout:Install 4 Packages 2024-09-13T14:00:26.939 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:00:26.940 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 79 M 2024-09-13T14:00:26.940 INFO:teuthology.orchestra.run.smithi145.stdout:Installed size: 126 M 2024-09-13T14:00:26.940 INFO:teuthology.orchestra.run.smithi145.stdout:Downloading Packages: 2024-09-13T14:00:27.575 INFO:teuthology.orchestra.run.smithi145.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 42 kB/s | 23 kB 00:00 2024-09-13T14:00:28.450 INFO:teuthology.orchestra.run.smithi145.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-13T14:00:29.184 INFO:teuthology.orchestra.run.smithi145.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 16 MB/s | 35 MB 00:02 2024-09-13T14:00:29.494 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 44 kB/s | 1.7 kB 00:00 2024-09-13T14:00:29.892 INFO:teuthology.orchestra.run.smithi145.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 12 MB/s | 29 MB 00:02 2024-09-13T14:00:29.892 INFO:teuthology.orchestra.run.smithi145.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:00:29.893 INFO:teuthology.orchestra.run.smithi145.stdout:Total 27 MB/s | 79 MB 00:02 2024-09-13T14:00:30.575 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check 2024-09-13T14:00:30.942 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded. 2024-09-13T14:00:30.942 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test 2024-09-13T14:00:31.559 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:00:31.559 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:00:31.657 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:00:31.662 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:Install 4 Packages 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 79 M 2024-09-13T14:00:31.663 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 126 M 2024-09-13T14:00:31.664 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2024-09-13T14:00:31.900 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded. 2024-09-13T14:00:31.900 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction 2024-09-13T14:00:32.004 INFO:teuthology.orchestra.run.smithi026.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 270 kB/s | 23 kB 00:00 2024-09-13T14:00:34.331 INFO:teuthology.orchestra.run.smithi145.stdout: Preparing : 1/1 2024-09-13T14:00:35.132 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:00:35.226 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:00:35.853 INFO:teuthology.orchestra.run.smithi026.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 4.0 MB/s | 16 MB 00:03 2024-09-13T14:00:38.011 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:00:38.147 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:00:39.361 INFO:teuthology.orchestra.run.smithi026.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 3.9 MB/s | 29 MB 00:07 2024-09-13T14:00:41.336 INFO:teuthology.orchestra.run.smithi026.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 3.7 MB/s | 35 MB 00:09 2024-09-13T14:00:41.337 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:00:41.337 INFO:teuthology.orchestra.run.smithi026.stdout:Total 8.1 MB/s | 79 MB 00:09 2024-09-13T14:00:42.050 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-09-13T14:00:42.426 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-09-13T14:00:42.426 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-09-13T14:00:42.999 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:00:43.255 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:00:43.414 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-09-13T14:00:43.414 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-09-13T14:00:45.987 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-09-13T14:00:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:00:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:00:48.120 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:00:49.539 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:00:49.663 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:00:54.781 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:00:54.822 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:00:59.962 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:26.552 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:26.592 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:26.931 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:26.932 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:01:26.932 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:01:26.932 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:01:27.800 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout:Installed: 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:01:27.801 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-09-13T14:01:28.184 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:01:28.268 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:28.268 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:01:28.268 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:01:28.268 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:01:28.268 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:01:28.269 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T14:01:28.269 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi145.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:01:28.269 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:01:28.327 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:28.327 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:01:28.328 DEBUG:teuthology.orchestra.run.smithi145:> sudo rpm -qi grub2-tools 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Name : grub2-tools 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Epoch : 1 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Version : 2.06 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Release : 82.el9 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Architecture: x86_64 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Group : Unspecified 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Size : 8274670 2024-09-13T14:01:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:License : GPLv3+ 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Packager : builder@centos.org 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Vendor : CentOS 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Summary : Support tools for GRUB. 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:Description : 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:01:28.368 INFO:teuthology.orchestra.run.smithi145.stdout:hardware devices. 2024-09-13T14:01:28.369 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:01:28.369 INFO:teuthology.orchestra.run.smithi145.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:01:28.370 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:01:28.370 INFO:teuthology.task.kernel:Updating grub on smithi145 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:01:28.370 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:01:29.013 INFO:teuthology.orchestra.run.smithi145.stderr:Generating grub configuration file ... 2024-09-13T14:01:30.317 INFO:teuthology.orchestra.run.smithi145.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:01:30.347 INFO:teuthology.orchestra.run.smithi145.stderr:done 2024-09-13T14:01:30.349 DEBUG:teuthology.orchestra.run.smithi145:> mktemp 2024-09-13T14:01:30.366 INFO:teuthology.orchestra.run.smithi145.stdout:/tmp/tmp.lw7vVwDiQq 2024-09-13T14:01:30.366 DEBUG:teuthology.orchestra.run.smithi145:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.lw7vVwDiQq 2024-09-13T14:01:30.436 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0666 /tmp/tmp.lw7vVwDiQq 2024-09-13T14:01:30.574 DEBUG:teuthology.orchestra.remote:smithi145:/tmp/tmp.lw7vVwDiQq is 6KB 2024-09-13T14:01:30.625 DEBUG:teuthology.orchestra.run.smithi145:> rm -fr /tmp/tmp.lw7vVwDiQq 2024-09-13T14:01:30.640 DEBUG:teuthology.orchestra.run.smithi145:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:01:30.703 INFO:teuthology.orchestra.run.smithi145.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:01:30.703 INFO:teuthology.orchestra.run.smithi145.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:01:30.703 INFO:teuthology.orchestra.run.smithi145.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:01:30.703 INFO:teuthology.orchestra.run.smithi145.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:01:30.705 DEBUG:teuthology.orchestra.run.smithi145:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:01:30.856 DEBUG:teuthology.orchestra.run.smithi145:> sudo shutdown -r now 2024-09-13T14:01:39.139 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:39.506 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:39.506 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:01:39.506 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:01:39.506 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:01:40.470 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-09-13T14:01:40.864 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:01:40.956 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:40.957 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:01:40.957 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:01:40.957 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:01:40.957 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:01:40.957 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T14:01:40.957 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi026.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:01:40.957 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:01:41.017 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:01:41.017 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:01:41.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo rpm -qi grub2-tools 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Name : grub2-tools 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Epoch : 1 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Version : 2.06 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Release : 82.el9 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Architecture: x86_64 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Group : Unspecified 2024-09-13T14:01:41.052 INFO:teuthology.orchestra.run.smithi026.stdout:Size : 8274670 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:License : GPLv3+ 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Packager : builder@centos.org 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Vendor : CentOS 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Summary : Support tools for GRUB. 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:Description : 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:01:41.053 INFO:teuthology.orchestra.run.smithi026.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:01:41.054 INFO:teuthology.orchestra.run.smithi026.stdout:hardware devices. 2024-09-13T14:01:41.054 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:01:41.054 INFO:teuthology.orchestra.run.smithi026.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:01:41.055 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:01:41.055 INFO:teuthology.task.kernel:Updating grub on smithi026 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:01:41.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:01:41.700 INFO:teuthology.orchestra.run.smithi026.stderr:Generating grub configuration file ... 2024-09-13T14:01:43.072 INFO:teuthology.orchestra.run.smithi026.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:01:43.100 INFO:teuthology.orchestra.run.smithi026.stderr:done 2024-09-13T14:01:43.102 DEBUG:teuthology.orchestra.run.smithi026:> mktemp 2024-09-13T14:01:43.117 INFO:teuthology.orchestra.run.smithi026.stdout:/tmp/tmp.hMBAS8JJRI 2024-09-13T14:01:43.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.hMBAS8JJRI 2024-09-13T14:01:43.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0666 /tmp/tmp.hMBAS8JJRI 2024-09-13T14:01:43.327 DEBUG:teuthology.orchestra.remote:smithi026:/tmp/tmp.hMBAS8JJRI is 6KB 2024-09-13T14:01:43.375 DEBUG:teuthology.orchestra.run.smithi026:> rm -fr /tmp/tmp.hMBAS8JJRI 2024-09-13T14:01:43.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:01:43.451 INFO:teuthology.orchestra.run.smithi026.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:01:43.452 INFO:teuthology.orchestra.run.smithi026.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:01:43.452 INFO:teuthology.orchestra.run.smithi026.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:01:43.452 INFO:teuthology.orchestra.run.smithi026.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:01:43.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:01:43.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo shutdown -r now 2024-09-13T14:02:00.873 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T14:02:00.874 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:02:00.875 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:02:13.621 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T14:02:13.621 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:02:13.622 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:02:19.325 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145 2024-09-13T14:02:28.326 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:02:28.327 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:02:31.389 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145 2024-09-13T14:02:32.125 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.26 2024-09-13T14:02:41.134 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:02:41.135 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:02:43.392 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:02:43.393 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:02:59.521 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.26 2024-09-13T14:03:11.533 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:03:11.534 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:03:12.965 DEBUG:teuthology.orchestra.run.smithi026:> true 2024-09-13T14:03:13.371 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:03:13.371 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:03:13.372 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2024-09-13T14:03:13.388 INFO:teuthology.orchestra.run.smithi026.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:03:13.388 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:03:13.389 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:03:13.389 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:03:14.389 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T14:03:14.390 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:03:14.497 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1 2024-09-13T14:03:14.516 DEBUG:teuthology.parallel:result is None 2024-09-13T14:03:15.224 DEBUG:teuthology.orchestra.run.smithi145:> true 2024-09-13T14:03:15.628 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:03:15.628 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:03:15.629 DEBUG:teuthology.orchestra.run.smithi145:> uname -r 2024-09-13T14:03:15.643 INFO:teuthology.orchestra.run.smithi145.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:03:15.643 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:03:15.643 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:03:15.643 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:03:16.645 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T14:03:16.645 DEBUG:teuthology.orchestra.run.smithi145:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:03:16.738 INFO:teuthology.orchestra.run.smithi145.stdout:ttyS1 2024-09-13T14:03:16.763 DEBUG:teuthology.parallel:result is None 2024-09-13T14:03:16.763 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T14:03:16.770 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T14:03:16.770 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:03:16.774 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:03:16.789 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T14:03:16.814 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T14:03:16.847 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T14:03:16.847 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:03:16.850 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:03:16.880 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T14:03:16.886 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T14:03:16.887 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:03:16.908 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:03:16.935 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:03:16.940 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:03:16.946 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:03:16.948 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:03:16.950 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T14:03:16.960 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T14:03:16.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:03:16.992 DEBUG:teuthology.orchestra.run.smithi145:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:03:17.018 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T14:03:17.031 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T14:03:17.031 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:03:17.059 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:03:17.075 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:03:17.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:03:17.198 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:03:17.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:03:17.316 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:03:17.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:03:17.382 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:03:17.438 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:03:17.523 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:03:17.596 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:03:17.656 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:03:17.656 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:03:17.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart 2024-09-13T14:03:17.717 DEBUG:teuthology.orchestra.run.smithi145:> sudo service rsyslog restart 2024-09-13T14:03:17.763 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:03:17.851 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:03:18.206 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T14:03:18.214 INFO:teuthology.task.internal:Starting timer... 2024-09-13T14:03:18.214 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T14:03:18.235 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T14:03:18.268 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-13T14:03:18.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate 2024-09-13T14:03:18.340 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs: 2024-09-13T14:03:18.341 DEBUG:teuthology.orchestra.run.smithi145:> sudo service auditd rotate 2024-09-13T14:03:18.472 INFO:teuthology.orchestra.run.smithi145.stdout:Rotating logs: 2024-09-13T14:03:18.475 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T14:03:18.475 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce 2024-09-13T14:03:18.506 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive 2024-09-13T14:03:18.506 DEBUG:teuthology.orchestra.run.smithi145:> /usr/sbin/getenforce 2024-09-13T14:03:18.528 INFO:teuthology.orchestra.run.smithi145.stdout:Permissive 2024-09-13T14:03:18.528 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi145.front.sepia.ceph.com': 'permissive'} 2024-09-13T14:03:18.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:03:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:03:18.570 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:03:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:03:18.595 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T14:03:18.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:03:18.698 DEBUG:teuthology.orchestra.run.smithi145:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:03:18.863 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T14:03:18.874 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-13T14:03:18.877 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-13T14:03:19.024 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T14:03:19.039 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T14:03:19.040 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi145.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T14:07:05.762 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi026.front.sepia.ceph.com'), Remote(name='ubuntu@smithi145.front.sepia.ceph.com')] 2024-09-13T14:07:05.763 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:07:05.764 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:07:05.849 DEBUG:teuthology.orchestra.run.smithi026:> true 2024-09-13T14:07:05.928 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-09-13T14:07:05.929 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:07:05.929 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:07:06.013 DEBUG:teuthology.orchestra.run.smithi145:> true 2024-09-13T14:07:06.091 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-09-13T14:07:06.092 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T14:07:06.106 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T14:07:06.106 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:07:06.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:07:06.109 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:07:06.109 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:07:06.137 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:07:06.152 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:07:06.160 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:07:06.174 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:07:06.178 INFO:teuthology.orchestra.run.smithi026.stderr:sudo: ntpd: command not found 2024-09-13T14:07:06.189 INFO:teuthology.orchestra.run.smithi026.stdout:506 Cannot talk to daemon 2024-09-13T14:07:06.198 INFO:teuthology.orchestra.run.smithi145.stderr:sudo: ntpd: command not found 2024-09-13T14:07:06.204 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:07:06.209 INFO:teuthology.orchestra.run.smithi145.stdout:506 Cannot talk to daemon 2024-09-13T14:07:06.219 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:07:06.223 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:07:06.237 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:07:06.272 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:07:06.276 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:07:06.276 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-09-13T14:07:06.276 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.276 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.277 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.277 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.292 INFO:teuthology.orchestra.run.smithi145.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:=============================================================================== 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.295 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:07:06.296 INFO:teuthology.run_tasks:Running task pexec... 2024-09-13T14:07:06.306 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-13T14:07:06.307 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi026.front.sepia.ceph.com 2024-09-13T14:07:06.307 DEBUG:teuthology.orchestra.run.smithi026:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:07:06.308 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi145.front.sepia.ceph.com 2024-09-13T14:07:06.308 DEBUG:teuthology.orchestra.run.smithi145:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:07:06.686 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-09-13T14:07:06.686 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout: Package Architecture Version Repository Size 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:Removing: 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:Removing unused dependencies: 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:06.698 INFO:teuthology.orchestra.run.smithi145.stdout:Remove 2 Packages 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi145.stdout:Freed space: 5.5 M 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-09-13T14:07:06.699 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Removing: 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Removing unused dependencies: 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Remove 2 Packages 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Freed space: 5.5 M 2024-09-13T14:07:06.700 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-09-13T14:07:06.702 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded. 2024-09-13T14:07:06.702 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test 2024-09-13T14:07:06.703 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-09-13T14:07:06.704 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-09-13T14:07:06.763 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-09-13T14:07:06.764 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-09-13T14:07:06.766 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded. 2024-09-13T14:07:06.767 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction 2024-09-13T14:07:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-09-13T14:07:06.912 INFO:teuthology.orchestra.run.smithi145.stdout: Preparing : 1/1 2024-09-13T14:07:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:07:06.984 INFO:teuthology.orchestra.run.smithi145.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:07:07.041 INFO:teuthology.orchestra.run.smithi026.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:07:07.044 INFO:teuthology.orchestra.run.smithi145.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:07:07.512 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:07:07.512 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:07:07.541 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:07:07.542 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout:Removed: 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:07.757 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout:Removed: 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:07.845 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-09-13T14:07:08.643 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:46 ago on Fri 13 Sep 2024 02:06:22 PM UTC. 2024-09-13T14:07:08.748 INFO:teuthology.orchestra.run.smithi145.stdout:Last metadata expiration check: 0:00:51 ago on Fri 13 Sep 2024 02:06:17 PM UTC. 2024-09-13T14:07:08.796 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:07:08.797 INFO:teuthology.orchestra.run.smithi026.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout:Install 7 Packages 2024-09-13T14:07:08.798 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:08.819 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 5.2 M 2024-09-13T14:07:08.819 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 19 M 2024-09-13T14:07:08.819 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2024-09-13T14:07:08.886 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: Package Arch Version Repository Size 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout:Installing: 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout:Installing dependencies: 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:07:08.887 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:08.888 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-09-13T14:07:08.888 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-09-13T14:07:08.888 INFO:teuthology.orchestra.run.smithi145.stdout:Install 7 Packages 2024-09-13T14:07:08.888 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:08.898 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 5.2 M 2024-09-13T14:07:08.899 INFO:teuthology.orchestra.run.smithi145.stdout:Installed size: 19 M 2024-09-13T14:07:08.899 INFO:teuthology.orchestra.run.smithi145.stdout:Downloading Packages: 2024-09-13T14:07:09.345 INFO:teuthology.orchestra.run.smithi026.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 300 kB/s | 44 kB 00:00 2024-09-13T14:07:09.372 INFO:teuthology.orchestra.run.smithi026.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 611 kB/s | 106 kB 00:00 2024-09-13T14:07:09.404 INFO:teuthology.orchestra.run.smithi026.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-13T14:07:09.463 INFO:teuthology.orchestra.run.smithi026.stdout:(4/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.3 MB/s | 906 kB 00:00 2024-09-13T14:07:09.488 INFO:teuthology.orchestra.run.smithi026.stdout:(5/7): python3-kmod-0.9-32.el9.x86_64.rpm 718 kB/s | 84 kB 00:00 2024-09-13T14:07:09.517 INFO:teuthology.orchestra.run.smithi145.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 187 kB/s | 44 kB 00:00 2024-09-13T14:07:09.542 INFO:teuthology.orchestra.run.smithi145.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 403 kB/s | 106 kB 00:00 2024-09-13T14:07:09.546 INFO:teuthology.orchestra.run.smithi026.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 5.8 MB/s | 837 kB 00:00 2024-09-13T14:07:09.659 INFO:teuthology.orchestra.run.smithi145.stdout:(3/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.3 MB/s | 906 kB 00:00 2024-09-13T14:07:09.719 INFO:teuthology.orchestra.run.smithi145.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 474 kB/s | 84 kB 00:00 2024-09-13T14:07:09.753 INFO:teuthology.orchestra.run.smithi145.stdout:(5/7): python3-configshell-1.1.30-1.el9.noarch. 304 kB/s | 72 kB 00:00 2024-09-13T14:07:09.787 INFO:teuthology.orchestra.run.smithi145.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 6.4 MB/s | 837 kB 00:00 2024-09-13T14:07:10.613 INFO:teuthology.orchestra.run.smithi026.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 2.8 MB/s | 3.2 MB 00:01 2024-09-13T14:07:10.614 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:07:10.614 INFO:teuthology.orchestra.run.smithi026.stdout:Total 2.9 MB/s | 5.2 MB 00:01 2024-09-13T14:07:10.734 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-09-13T14:07:10.750 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-09-13T14:07:10.750 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-09-13T14:07:11.086 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-09-13T14:07:11.087 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-09-13T14:07:11.253 INFO:teuthology.orchestra.run.smithi145.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 2.1 MB/s | 3.2 MB 00:01 2024-09-13T14:07:11.253 INFO:teuthology.orchestra.run.smithi145.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:07:11.254 INFO:teuthology.orchestra.run.smithi145.stdout:Total 2.2 MB/s | 5.2 MB 00:02 2024-09-13T14:07:11.372 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check 2024-09-13T14:07:11.387 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded. 2024-09-13T14:07:11.387 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test 2024-09-13T14:07:11.579 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-09-13T14:07:11.669 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:07:11.703 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:07:11.703 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded. 2024-09-13T14:07:11.704 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction 2024-09-13T14:07:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:07:11.956 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:07:11.987 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:07:12.161 INFO:teuthology.orchestra.run.smithi145.stdout: Preparing : 1/1 2024-09-13T14:07:12.239 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:07:12.274 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:07:12.302 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:07:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:07:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:07:12.516 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:07:12.550 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:07:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:07:12.704 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:12.998 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:07:13.028 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:07:13.143 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:07:13.206 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:07:13.748 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:07:14.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:14.121 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-09-13T14:07:14.197 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:14.197 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:07:14.197 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:07:14.197 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:07:14.197 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:07:14.198 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:07:14.198 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:07:14.489 DEBUG:teuthology.parallel:result is None 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout:Installed: 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:14.581 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-09-13T14:07:14.953 DEBUG:teuthology.parallel:result is None 2024-09-13T14:07:14.953 INFO:teuthology.run_tasks:Running task install... 2024-09-13T14:07:14.963 DEBUG:teuthology.task.install:project ceph 2024-09-13T14:07:14.963 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:07:14.963 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:07:14.963 INFO:teuthology.task.install:Using flavor: default 2024-09-13T14:07:14.969 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T14:07:14.969 INFO:teuthology.task.install:extra packages: [] 2024-09-13T14:07:14.969 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:07:14.969 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:14.970 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:14.970 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:14.970 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:14.970 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:14.970 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:07:14.972 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:07:14.972 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:14.972 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:14.972 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:14.972 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:14.973 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:14.973 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:07:15.198 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:07:15.198 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:07:15.213 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:07:15.214 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:07:15.320 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:07:15.320 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:07:15.320 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:07:15.335 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:07:15.335 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:07:15.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:07:15.347 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-13T14:07:15.347 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:15.347 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:15.347 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:15.347 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:15.347 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:15.348 DEBUG:teuthology.orchestra.run.smithi145:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:07:15.369 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-13T14:07:15.370 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:15.370 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:15.370 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:15.370 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:15.370 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:15.370 DEBUG:teuthology.orchestra.run.smithi026:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:07:15.418 DEBUG:teuthology.orchestra.run.smithi145:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:07:15.439 DEBUG:teuthology.orchestra.run.smithi026:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:07:15.494 DEBUG:teuthology.orchestra.run.smithi145:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:07:15.520 DEBUG:teuthology.orchestra.run.smithi026:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:07:15.558 INFO:teuthology.orchestra.run.smithi145.stdout:check_obsoletes = 1 2024-09-13T14:07:15.560 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum clean all 2024-09-13T14:07:15.587 INFO:teuthology.orchestra.run.smithi026.stdout:check_obsoletes = 1 2024-09-13T14:07:15.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum clean all 2024-09-13T14:07:15.793 INFO:teuthology.orchestra.run.smithi145.stdout:45 files removed 2024-09-13T14:07:15.820 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-13T14:07:15.828 INFO:teuthology.orchestra.run.smithi026.stdout:45 files removed 2024-09-13T14:07:15.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-13T14:07:16.393 INFO:teuthology.orchestra.run.smithi145.stdout:ceph packages for x86_64 277 kB/s | 77 kB 00:00 2024-09-13T14:07:16.439 INFO:teuthology.orchestra.run.smithi026.stdout:ceph packages for x86_64 276 kB/s | 77 kB 00:00 2024-09-13T14:07:16.677 INFO:teuthology.orchestra.run.smithi145.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-13T14:07:16.723 INFO:teuthology.orchestra.run.smithi026.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-13T14:07:16.952 INFO:teuthology.orchestra.run.smithi145.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-13T14:07:16.998 INFO:teuthology.orchestra.run.smithi026.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-13T14:07:17.478 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-13T14:07:17.498 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - BaseOS 18 MB/s | 8.2 MB 00:00 2024-09-13T14:07:19.154 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - AppStream 35 MB/s | 20 MB 00:00 2024-09-13T14:07:19.291 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-09-13T14:07:23.634 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - CRB 17 MB/s | 6.5 MB 00:00 2024-09-13T14:07:23.712 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - CRB 9.4 MB/s | 6.5 MB 00:00 2024-09-13T14:07:25.110 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - Extras packages 104 kB/s | 19 kB 00:00 2024-09-13T14:07:25.363 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-13T14:07:26.930 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 13 MB/s | 23 MB 00:01 2024-09-13T14:07:27.126 INFO:teuthology.orchestra.run.smithi145.stdout:Extra Packages for Enterprise Linux 14 MB/s | 23 MB 00:01 2024-09-13T14:07:32.448 INFO:teuthology.orchestra.run.smithi145.stdout:lab-extras 20 kB/s | 1.7 kB 00:00 2024-09-13T14:07:32.653 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-13T14:07:34.767 INFO:teuthology.orchestra.run.smithi145.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-13T14:07:34.768 INFO:teuthology.orchestra.run.smithi145.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-13T14:07:34.843 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-09-13T14:07:34.849 INFO:teuthology.orchestra.run.smithi145.stdout:======================================================================================= 2024-09-13T14:07:34.849 INFO:teuthology.orchestra.run.smithi145.stdout: Package Arch Version Repository Size 2024-09-13T14:07:34.849 INFO:teuthology.orchestra.run.smithi145.stdout:======================================================================================= 2024-09-13T14:07:34.849 INFO:teuthology.orchestra.run.smithi145.stdout:Installing: 2024-09-13T14:07:34.849 INFO:teuthology.orchestra.run.smithi145.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:07:34.850 INFO:teuthology.orchestra.run.smithi145.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:07:34.851 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout:Upgrading: 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout:Installing dependencies: 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:07:34.852 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:07:34.853 INFO:teuthology.orchestra.run.smithi145.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:07:34.854 INFO:teuthology.orchestra.run.smithi145.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:07:34.855 INFO:teuthology.orchestra.run.smithi145.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:07:34.856 INFO:teuthology.orchestra.run.smithi145.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:07:34.857 INFO:teuthology.orchestra.run.smithi145.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:07:34.858 INFO:teuthology.orchestra.run.smithi145.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout:Installing weak dependencies: 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:34.859 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout:======================================================================================= 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout:Install 112 Packages 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout:Upgrade 2 Packages 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 190 M 2024-09-13T14:07:34.860 INFO:teuthology.orchestra.run.smithi145.stdout:Downloading Packages: 2024-09-13T14:07:35.065 INFO:teuthology.orchestra.run.smithi026.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-13T14:07:35.066 INFO:teuthology.orchestra.run.smithi026.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-13T14:07:35.139 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-09-13T14:07:35.146 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================= 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================= 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:07:35.147 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:07:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrading: 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:07:35.149 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:07:35.150 INFO:teuthology.orchestra.run.smithi026.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:07:35.151 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:07:35.152 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:07:35.153 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:07:35.154 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:07:35.155 INFO:teuthology.orchestra.run.smithi026.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout:Installing weak dependencies: 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-09-13T14:07:35.156 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================= 2024-09-13T14:07:35.157 INFO:teuthology.orchestra.run.smithi026.stdout:Install 112 Packages 2024-09-13T14:07:35.157 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrade 2 Packages 2024-09-13T14:07:35.157 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:07:35.157 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 190 M 2024-09-13T14:07:35.157 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2024-09-13T14:07:35.741 INFO:teuthology.orchestra.run.smithi145.stdout:(1/114): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64. 49 kB/s | 6.5 kB 00:00 2024-09-13T14:07:35.829 INFO:teuthology.orchestra.run.smithi026.stdout:(1/114): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64. 55 kB/s | 6.5 kB 00:00 2024-09-13T14:07:35.900 INFO:teuthology.orchestra.run.smithi145.stdout:(2/114): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x8 5.0 MB/s | 813 kB 00:00 2024-09-13T14:07:36.008 INFO:teuthology.orchestra.run.smithi145.stdout:(3/114): ceph-immutable-object-cache-17.2.7-162 1.2 MB/s | 137 kB 00:00 2024-09-13T14:07:36.229 INFO:teuthology.orchestra.run.smithi026.stdout:(2/114): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x8 2.0 MB/s | 813 kB 00:00 2024-09-13T14:07:36.258 INFO:teuthology.orchestra.run.smithi145.stdout:(4/114): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86 8.3 MB/s | 2.1 MB 00:00 2024-09-13T14:07:36.305 INFO:teuthology.orchestra.run.smithi026.stdout:(3/114): ceph-immutable-object-cache-17.2.7-162 1.8 MB/s | 137 kB 00:00 2024-09-13T14:07:36.334 INFO:teuthology.orchestra.run.smithi145.stdout:(5/114): ceph-base-17.2.7-1627.gbbdadfa3.el9.x8 6.9 MB/s | 5.0 MB 00:00 2024-09-13T14:07:36.426 INFO:teuthology.orchestra.run.smithi145.stdout:(6/114): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86 8.4 MB/s | 1.4 MB 00:00 2024-09-13T14:07:36.647 INFO:teuthology.orchestra.run.smithi026.stdout:(4/114): ceph-base-17.2.7-1627.gbbdadfa3.el9.x8 5.3 MB/s | 5.0 MB 00:00 2024-09-13T14:07:36.780 INFO:teuthology.orchestra.run.smithi026.stdout:(5/114): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86 11 MB/s | 1.4 MB 00:00 2024-09-13T14:07:36.792 INFO:teuthology.orchestra.run.smithi145.stdout:(7/114): ceph-common-17.2.7-1627.gbbdadfa3.el9. 19 MB/s | 22 MB 00:01 2024-09-13T14:07:36.832 INFO:teuthology.orchestra.run.smithi026.stdout:(6/114): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86 3.9 MB/s | 2.1 MB 00:00 2024-09-13T14:07:37.334 INFO:teuthology.orchestra.run.smithi145.stdout:(8/114): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x86 17 MB/s | 15 MB 00:00 2024-09-13T14:07:37.389 INFO:teuthology.orchestra.run.smithi026.stdout:(7/114): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86 6.4 MB/s | 3.9 MB 00:00 2024-09-13T14:07:37.401 INFO:teuthology.orchestra.run.smithi145.stdout:(9/114): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86 3.7 MB/s | 3.9 MB 00:01 2024-09-13T14:07:37.426 INFO:teuthology.orchestra.run.smithi145.stdout:(10/114): ceph-selinux-17.2.7-1627.gbbdadfa3.el 274 kB/s | 25 kB 00:00 2024-09-13T14:07:37.502 INFO:teuthology.orchestra.run.smithi145.stdout:(11/114): libcephfs-devel-17.2.7-1627.gbbdadfa3 332 kB/s | 25 kB 00:00 2024-09-13T14:07:37.594 INFO:teuthology.orchestra.run.smithi145.stdout:(12/114): libcephfs2-17.2.7-1627.gbbdadfa3.el9. 6.8 MB/s | 637 kB 00:00 2024-09-13T14:07:37.677 INFO:teuthology.orchestra.run.smithi145.stdout:(13/114): libcephsqlite-17.2.7-1627.gbbdadfa3.e 1.9 MB/s | 163 kB 00:00 2024-09-13T14:07:37.753 INFO:teuthology.orchestra.run.smithi145.stdout:(14/114): librados-devel-17.2.7-1627.gbbdadfa3. 1.6 MB/s | 126 kB 00:00 2024-09-13T14:07:37.811 INFO:teuthology.orchestra.run.smithi145.stdout:(15/114): libradosstriper1-17.2.7-1627.gbbdadfa 7.7 MB/s | 457 kB 00:00 2024-09-13T14:07:37.931 INFO:teuthology.orchestra.run.smithi026.stdout:(8/114): ceph-common-17.2.7-1627.gbbdadfa3.el9. 10 MB/s | 22 MB 00:02 2024-09-13T14:07:37.961 INFO:teuthology.orchestra.run.smithi145.stdout:(16/114): ceph-radosgw-17.2.7-1627.gbbdadfa3.el 12 MB/s | 14 MB 00:01 2024-09-13T14:07:37.981 INFO:teuthology.orchestra.run.smithi026.stdout:(9/114): ceph-selinux-17.2.7-1627.gbbdadfa3.el9 506 kB/s | 25 kB 00:00 2024-09-13T14:07:38.020 INFO:teuthology.orchestra.run.smithi145.stdout:(17/114): python3-ceph-argparse-17.2.7-1627.gbb 776 kB/s | 45 kB 00:00 2024-09-13T14:07:38.087 INFO:teuthology.orchestra.run.smithi145.stdout:(18/114): librgw2-17.2.7-1627.gbbdadfa3.el9.x86 18 MB/s | 4.9 MB 00:00 2024-09-13T14:07:38.112 INFO:teuthology.orchestra.run.smithi145.stdout:(19/114): python3-ceph-common-17.2.7-1627.gbbda 1.3 MB/s | 120 kB 00:00 2024-09-13T14:07:38.154 INFO:teuthology.orchestra.run.smithi145.stdout:(20/114): python3-cephfs-17.2.7-1627.gbbdadfa3. 2.3 MB/s | 156 kB 00:00 2024-09-13T14:07:38.188 INFO:teuthology.orchestra.run.smithi145.stdout:(21/114): python3-rados-17.2.7-1627.gbbdadfa3.e 4.2 MB/s | 320 kB 00:00 2024-09-13T14:07:38.230 INFO:teuthology.orchestra.run.smithi145.stdout:(22/114): python3-rbd-17.2.7-1627.gbbdadfa3.el9 3.8 MB/s | 294 kB 00:00 2024-09-13T14:07:38.263 INFO:teuthology.orchestra.run.smithi145.stdout:(23/114): python3-rgw-17.2.7-1627.gbbdadfa3.el9 1.3 MB/s | 99 kB 00:00 2024-09-13T14:07:38.297 INFO:teuthology.orchestra.run.smithi145.stdout:(24/114): rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x8 1.2 MB/s | 82 kB 00:00 2024-09-13T14:07:38.389 INFO:teuthology.orchestra.run.smithi145.stdout:(25/114): rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86 1.8 MB/s | 165 kB 00:00 2024-09-13T14:07:38.481 INFO:teuthology.orchestra.run.smithi145.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1627.g 267 kB/s | 24 kB 00:00 2024-09-13T14:07:38.556 INFO:teuthology.orchestra.run.smithi145.stdout:(27/114): rbd-mirror-17.2.7-1627.gbbdadfa3.el9. 10 MB/s | 2.9 MB 00:00 2024-09-13T14:07:38.581 INFO:teuthology.orchestra.run.smithi145.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa 1.2 MB/s | 124 kB 00:00 2024-09-13T14:07:38.898 INFO:teuthology.orchestra.run.smithi145.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1627.gbbdad 9.8 MB/s | 3.4 MB 00:00 2024-09-13T14:07:38.998 INFO:teuthology.orchestra.run.smithi145.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1627.gbb 2.4 MB/s | 240 kB 00:00 2024-09-13T14:07:39.090 INFO:teuthology.orchestra.run.smithi145.stdout:(31/114): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.e 516 kB/s | 47 kB 00:00 2024-09-13T14:07:39.208 INFO:teuthology.orchestra.run.smithi145.stdout:(32/114): ceph-mgr-diskprediction-local-17.2.7- 12 MB/s | 7.4 MB 00:00 2024-09-13T14:07:39.233 INFO:teuthology.orchestra.run.smithi145.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1627.gb 103 kB/s | 15 kB 00:00 2024-09-13T14:07:39.274 INFO:teuthology.orchestra.run.smithi145.stdout:(34/114): cephadm-17.2.7-1627.gbbdadfa3.el9.noa 1.2 MB/s | 83 kB 00:00 2024-09-13T14:07:39.383 INFO:teuthology.orchestra.run.smithi145.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 532 kB/s | 80 kB 00:00 2024-09-13T14:07:39.467 INFO:teuthology.orchestra.run.smithi145.stdout:(36/114): libquadmath-11.5.0-2.el9.x86_64.rpm 2.2 MB/s | 191 kB 00:00 2024-09-13T14:07:39.523 INFO:teuthology.orchestra.run.smithi026.stdout:(10/114): ceph-radosgw-17.2.7-1627.gbbdadfa3.el 6.8 MB/s | 14 MB 00:02 2024-09-13T14:07:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:(37/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.2 MB/s | 253 kB 00:00 2024-09-13T14:07:39.582 INFO:teuthology.orchestra.run.smithi026.stdout:(11/114): libcephfs-devel-17.2.7-1627.gbbdadfa3 427 kB/s | 25 kB 00:00 2024-09-13T14:07:39.592 INFO:teuthology.orchestra.run.smithi145.stdout:(38/114): libgfortran-11.5.0-2.el9.x86_64.rpm 2.5 MB/s | 801 kB 00:00 2024-09-13T14:07:39.651 INFO:teuthology.orchestra.run.smithi145.stdout:(39/114): python3-ply-3.11-14.el9.noarch.rpm 1.8 MB/s | 106 kB 00:00 2024-09-13T14:07:39.691 INFO:teuthology.orchestra.run.smithi026.stdout:(12/114): libcephfs2-17.2.7-1627.gbbdadfa3.el9. 5.7 MB/s | 637 kB 00:00 2024-09-13T14:07:39.710 INFO:teuthology.orchestra.run.smithi145.stdout:(40/114): python3-cryptography-36.0.1-4.el9.x86 6.7 MB/s | 1.2 MB 00:00 2024-09-13T14:07:39.735 INFO:teuthology.orchestra.run.smithi145.stdout:(41/114): python3-pycparser-2.20-6.el9.noarch.r 1.6 MB/s | 135 kB 00:00 2024-09-13T14:07:39.749 INFO:teuthology.orchestra.run.smithi026.stdout:(13/114): libcephsqlite-17.2.7-1627.gbbdadfa3.e 2.7 MB/s | 163 kB 00:00 2024-09-13T14:07:39.760 INFO:teuthology.orchestra.run.smithi145.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 2.4 MB/s | 125 kB 00:00 2024-09-13T14:07:39.794 INFO:teuthology.orchestra.run.smithi145.stdout:(43/114): python3-urllib3-1.26.5-6.el9.noarch.r 3.6 MB/s | 215 kB 00:00 2024-09-13T14:07:39.808 INFO:teuthology.orchestra.run.smithi026.stdout:(14/114): librados-devel-17.2.7-1627.gbbdadfa3. 2.1 MB/s | 126 kB 00:00 2024-09-13T14:07:39.861 INFO:teuthology.orchestra.run.smithi145.stdout:(44/114): flexiblas-3.0.4-8.el9.x86_64.rpm 326 kB/s | 33 kB 00:00 2024-09-13T14:07:39.911 INFO:teuthology.orchestra.run.smithi145.stdout:(45/114): flexiblas-openblas-openmp-3.0.4-8.el9 365 kB/s | 18 kB 00:00 2024-09-13T14:07:39.995 INFO:teuthology.orchestra.run.smithi145.stdout:(46/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.9 MB/s | 160 kB 00:00 2024-09-13T14:07:40.054 INFO:teuthology.orchestra.run.smithi145.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 775 kB/s | 45 kB 00:00 2024-09-13T14:07:40.279 INFO:teuthology.orchestra.run.smithi145.stdout:(48/114): librdkafka-1.6.1-102.el9.x86_64.rpm 2.9 MB/s | 662 kB 00:00 2024-09-13T14:07:40.283 INFO:teuthology.orchestra.run.smithi026.stdout:(15/114): ceph-test-17.2.7-1627.gbbdadfa3.el9.x 21 MB/s | 48 MB 00:02 2024-09-13T14:07:40.316 INFO:teuthology.orchestra.run.smithi026.stdout:(16/114): libradosstriper1-17.2.7-1627.gbbdadfa 899 kB/s | 457 kB 00:00 2024-09-13T14:07:40.367 INFO:teuthology.orchestra.run.smithi026.stdout:(17/114): python3-ceph-argparse-17.2.7-1627.gbb 907 kB/s | 45 kB 00:00 2024-09-13T14:07:40.396 INFO:teuthology.orchestra.run.smithi145.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.5 MB/s | 292 kB 00:00 2024-09-13T14:07:40.433 INFO:teuthology.orchestra.run.smithi026.stdout:(18/114): python3-ceph-common-17.2.7-1627.gbbda 1.8 MB/s | 120 kB 00:00 2024-09-13T14:07:40.446 INFO:teuthology.orchestra.run.smithi145.stdout:(50/114): openblas-0.3.26-2.el9.x86_64.rpm 769 kB/s | 38 kB 00:00 2024-09-13T14:07:40.609 INFO:teuthology.orchestra.run.smithi026.stdout:(19/114): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x8 4.1 MB/s | 15 MB 00:03 2024-09-13T14:07:40.684 INFO:teuthology.orchestra.run.smithi026.stdout:(20/114): librgw2-17.2.7-1627.gbbdadfa3.el9.x86 12 MB/s | 4.9 MB 00:00 2024-09-13T14:07:40.709 INFO:teuthology.orchestra.run.smithi026.stdout:(21/114): python3-cephfs-17.2.7-1627.gbbdadfa3. 566 kB/s | 156 kB 00:00 2024-09-13T14:07:40.729 INFO:teuthology.orchestra.run.smithi145.stdout:(51/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 3.2 MB/s | 3.0 MB 00:00 2024-09-13T14:07:40.735 INFO:teuthology.orchestra.run.smithi026.stdout:(22/114): python3-rados-17.2.7-1627.gbbdadfa3.e 2.5 MB/s | 320 kB 00:00 2024-09-13T14:07:40.760 INFO:teuthology.orchestra.run.smithi026.stdout:(23/114): python3-rbd-17.2.7-1627.gbbdadfa3.el9 3.8 MB/s | 294 kB 00:00 2024-09-13T14:07:40.793 INFO:teuthology.orchestra.run.smithi026.stdout:(24/114): python3-rgw-17.2.7-1627.gbbdadfa3.el9 1.2 MB/s | 99 kB 00:00 2024-09-13T14:07:40.819 INFO:teuthology.orchestra.run.smithi026.stdout:(25/114): rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x8 1.4 MB/s | 82 kB 00:00 2024-09-13T14:07:40.852 INFO:teuthology.orchestra.run.smithi026.stdout:(26/114): rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-13T14:07:40.869 INFO:teuthology.orchestra.run.smithi026.stdout:(27/114): ceph-grafana-dashboards-17.2.7-1627.g 488 kB/s | 24 kB 00:00 2024-09-13T14:07:40.920 INFO:teuthology.orchestra.run.smithi026.stdout:(28/114): rbd-mirror-17.2.7-1627.gbbdadfa3.el9. 18 MB/s | 2.9 MB 00:00 2024-09-13T14:07:40.953 INFO:teuthology.orchestra.run.smithi026.stdout:(29/114): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa 1.2 MB/s | 124 kB 00:00 2024-09-13T14:07:41.020 INFO:teuthology.orchestra.run.smithi026.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1627.gbb 3.5 MB/s | 240 kB 00:00 2024-09-13T14:07:41.087 INFO:teuthology.orchestra.run.smithi026.stdout:(31/114): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.e 714 kB/s | 47 kB 00:00 2024-09-13T14:07:41.137 INFO:teuthology.orchestra.run.smithi026.stdout:(32/114): ceph-prometheus-alerts-17.2.7-1627.gb 292 kB/s | 15 kB 00:00 2024-09-13T14:07:41.146 INFO:teuthology.orchestra.run.smithi145.stdout:(52/114): ceph-test-17.2.7-1627.gbbdadfa3.el9.x 13 MB/s | 48 MB 00:03 2024-09-13T14:07:41.172 INFO:teuthology.orchestra.run.smithi145.stdout:(53/114): python3-attrs-20.3.0-7.el9.noarch.rpm 198 kB/s | 87 kB 00:00 2024-09-13T14:07:41.229 INFO:teuthology.orchestra.run.smithi026.stdout:(33/114): ceph-mgr-diskprediction-local-17.2.7- 24 MB/s | 7.4 MB 00:00 2024-09-13T14:07:41.255 INFO:teuthology.orchestra.run.smithi026.stdout:(34/114): cephadm-17.2.7-1627.gbbdadfa3.el9.noa 707 kB/s | 83 kB 00:00 2024-09-13T14:07:41.297 INFO:teuthology.orchestra.run.smithi145.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 1.9 MB/s | 245 kB 00:00 2024-09-13T14:07:41.347 INFO:teuthology.orchestra.run.smithi145.stdout:(55/114): python3-iniconfig-1.1.1-7.el9.noarch. 350 kB/s | 17 kB 00:00 2024-09-13T14:07:41.355 INFO:teuthology.orchestra.run.smithi026.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 637 kB/s | 80 kB 00:00 2024-09-13T14:07:41.455 INFO:teuthology.orchestra.run.smithi026.stdout:(36/114): ceph-mgr-dashboard-17.2.7-1627.gbbdad 5.7 MB/s | 3.4 MB 00:00 2024-09-13T14:07:41.456 INFO:teuthology.orchestra.run.smithi145.stdout:(56/114): python3-jinja2-2.11.3-6.el9.noarch.rp 2.2 MB/s | 248 kB 00:00 2024-09-13T14:07:41.480 INFO:teuthology.orchestra.run.smithi026.stdout:(37/114): libquadmath-11.5.0-2.el9.x86_64.rpm 1.5 MB/s | 191 kB 00:00 2024-09-13T14:07:41.506 INFO:teuthology.orchestra.run.smithi145.stdout:(57/114): python3-jsonpatch-1.21-16.el9.noarch. 513 kB/s | 26 kB 00:00 2024-09-13T14:07:41.506 INFO:teuthology.orchestra.run.smithi026.stdout:(38/114): libgfortran-11.5.0-2.el9.x86_64.rpm 3.1 MB/s | 801 kB 00:00 2024-09-13T14:07:41.540 INFO:teuthology.orchestra.run.smithi026.stdout:(39/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.9 MB/s | 253 kB 00:00 2024-09-13T14:07:41.556 INFO:teuthology.orchestra.run.smithi145.stdout:(58/114): python3-jsonpointer-2.0-4.el9.noarch. 384 kB/s | 19 kB 00:00 2024-09-13T14:07:41.573 INFO:teuthology.orchestra.run.smithi026.stdout:(40/114): python3-cryptography-36.0.1-4.el9.x86 13 MB/s | 1.2 MB 00:00 2024-09-13T14:07:41.607 INFO:teuthology.orchestra.run.smithi026.stdout:(41/114): python3-pycparser-2.20-6.el9.noarch.r 2.0 MB/s | 135 kB 00:00 2024-09-13T14:07:41.632 INFO:teuthology.orchestra.run.smithi026.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 2.1 MB/s | 125 kB 00:00 2024-09-13T14:07:41.657 INFO:teuthology.orchestra.run.smithi026.stdout:(43/114): python3-ply-3.11-14.el9.noarch.rpm 705 kB/s | 106 kB 00:00 2024-09-13T14:07:41.665 INFO:teuthology.orchestra.run.smithi145.stdout:(59/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.6 MB/s | 172 kB 00:00 2024-09-13T14:07:41.683 INFO:teuthology.orchestra.run.smithi026.stdout:(44/114): python3-urllib3-1.26.5-6.el9.noarch.r 2.8 MB/s | 215 kB 00:00 2024-09-13T14:07:41.766 INFO:teuthology.orchestra.run.smithi026.stdout:(45/114): flexiblas-openblas-openmp-3.0.4-8.el9 219 kB/s | 18 kB 00:00 2024-09-13T14:07:41.773 INFO:teuthology.orchestra.run.smithi145.stdout:(60/114): python3-markupsafe-1.1.1-12.el9.x86_6 321 kB/s | 35 kB 00:00 2024-09-13T14:07:41.791 INFO:teuthology.orchestra.run.smithi026.stdout:(46/114): flexiblas-3.0.4-8.el9.x86_64.rpm 205 kB/s | 33 kB 00:00 2024-09-13T14:07:41.866 INFO:teuthology.orchestra.run.smithi145.stdout:(61/114): openblas-openmp-0.3.26-2.el9.x86_64.r 3.5 MB/s | 4.9 MB 00:01 2024-09-13T14:07:41.875 INFO:teuthology.orchestra.run.smithi026.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 544 kB/s | 45 kB 00:00 2024-09-13T14:07:41.900 INFO:teuthology.orchestra.run.smithi026.stdout:(48/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-09-13T14:07:41.949 INFO:teuthology.orchestra.run.smithi145.stdout:(62/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.9 MB/s | 250 kB 00:00 2024-09-13T14:07:42.009 INFO:teuthology.orchestra.run.smithi026.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-13T14:07:42.049 INFO:teuthology.orchestra.run.smithi145.stdout:(63/114): python3-oauthlib-3.1.1-5.el9.noarch.r 2.2 MB/s | 222 kB 00:00 2024-09-13T14:07:42.092 INFO:teuthology.orchestra.run.smithi026.stdout:(50/114): librdkafka-1.6.1-102.el9.x86_64.rpm 3.0 MB/s | 662 kB 00:00 2024-09-13T14:07:42.099 INFO:teuthology.orchestra.run.smithi145.stdout:(64/114): python3-packaging-20.9-5.el9.noarch.r 1.5 MB/s | 77 kB 00:00 2024-09-13T14:07:42.143 INFO:teuthology.orchestra.run.smithi026.stdout:(51/114): openblas-0.3.26-2.el9.x86_64.rpm 289 kB/s | 38 kB 00:00 2024-09-13T14:07:42.150 INFO:teuthology.orchestra.run.smithi145.stdout:(65/114): python3-pluggy-0.13.1-7.el9.noarch.rp 826 kB/s | 41 kB 00:00 2024-09-13T14:07:42.200 INFO:teuthology.orchestra.run.smithi145.stdout:(66/114): python3-prettytable-0.7.2-27.el9.noar 843 kB/s | 42 kB 00:00 2024-09-13T14:07:42.226 INFO:teuthology.orchestra.run.smithi026.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 1.0 MB/s | 87 kB 00:00 2024-09-13T14:07:42.342 INFO:teuthology.orchestra.run.smithi145.stdout:(67/114): python3-py-1.10.0-6.el9.noarch.rpm 3.3 MB/s | 477 kB 00:00 2024-09-13T14:07:42.401 INFO:teuthology.orchestra.run.smithi145.stdout:(68/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.7 MB/s | 159 kB 00:00 2024-09-13T14:07:42.484 INFO:teuthology.orchestra.run.smithi145.stdout:(69/114): python3-pyasn1-modules-0.4.8-6.el9.no 3.3 MB/s | 279 kB 00:00 2024-09-13T14:07:42.502 INFO:teuthology.orchestra.run.smithi026.stdout:(53/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 3.5 MB/s | 3.0 MB 00:00 2024-09-13T14:07:42.585 INFO:teuthology.orchestra.run.smithi026.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 2.9 MB/s | 245 kB 00:00 2024-09-13T14:07:42.635 INFO:teuthology.orchestra.run.smithi145.stdout:(70/114): python3-pytest-6.2.2-7.el9.noarch.rpm 3.4 MB/s | 519 kB 00:00 2024-09-13T14:07:42.635 INFO:teuthology.orchestra.run.smithi026.stdout:(55/114): python3-iniconfig-1.1.1-7.el9.noarch. 350 kB/s | 17 kB 00:00 2024-09-13T14:07:42.677 INFO:teuthology.orchestra.run.smithi145.stdout:(71/114): python3-pytz-2021.1-5.el9.noarch.rpm 1.2 MB/s | 51 kB 00:00 2024-09-13T14:07:42.718 INFO:teuthology.orchestra.run.smithi145.stdout:(72/114): python3-requests-oauthlib-1.3.0-12.el 1.3 MB/s | 54 kB 00:00 2024-09-13T14:07:42.719 INFO:teuthology.orchestra.run.smithi026.stdout:(56/114): python3-jinja2-2.11.3-6.el9.noarch.rp 2.9 MB/s | 248 kB 00:00 2024-09-13T14:07:42.772 INFO:teuthology.orchestra.run.smithi026.stdout:(57/114): python3-jsonpatch-1.21-16.el9.noarch. 512 kB/s | 26 kB 00:00 2024-09-13T14:07:42.812 INFO:teuthology.orchestra.run.smithi026.stdout:(58/114): python3-jsonpointer-2.0-4.el9.noarch. 461 kB/s | 19 kB 00:00 2024-09-13T14:07:42.895 INFO:teuthology.orchestra.run.smithi026.stdout:(59/114): python3-mako-1.1.4-6.el9.noarch.rpm 2.0 MB/s | 172 kB 00:00 2024-09-13T14:07:42.937 INFO:teuthology.orchestra.run.smithi026.stdout:(60/114): python3-markupsafe-1.1.1-12.el9.x86_6 831 kB/s | 35 kB 00:00 2024-09-13T14:07:43.186 INFO:teuthology.orchestra.run.smithi145.stdout:(73/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.6 MB/s | 5.1 MB 00:01 2024-09-13T14:07:43.188 INFO:teuthology.orchestra.run.smithi026.stdout:(61/114): openblas-openmp-0.3.26-2.el9.x86_64.r 4.5 MB/s | 4.9 MB 00:01 2024-09-13T14:07:43.236 INFO:teuthology.orchestra.run.smithi145.stdout:(74/114): python3-toml-0.10.2-6.el9.noarch.rpm 836 kB/s | 42 kB 00:00 2024-09-13T14:07:43.271 INFO:teuthology.orchestra.run.smithi026.stdout:(62/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.9 MB/s | 250 kB 00:00 2024-09-13T14:07:43.345 INFO:teuthology.orchestra.run.smithi145.stdout:(75/114): socat-1.7.4.1-6.el9.x86_64.rpm 2.7 MB/s | 304 kB 00:00 2024-09-13T14:07:43.354 INFO:teuthology.orchestra.run.smithi026.stdout:(63/114): python3-oauthlib-3.1.1-5.el9.noarch.r 2.6 MB/s | 222 kB 00:00 2024-09-13T14:07:43.395 INFO:teuthology.orchestra.run.smithi145.stdout:(76/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.3 MB/s | 64 kB 00:00 2024-09-13T14:07:43.405 INFO:teuthology.orchestra.run.smithi026.stdout:(64/114): python3-packaging-20.9-5.el9.noarch.r 1.5 MB/s | 77 kB 00:00 2024-09-13T14:07:43.464 INFO:teuthology.orchestra.run.smithi026.stdout:(65/114): python3-pluggy-0.13.1-7.el9.noarch.rp 709 kB/s | 41 kB 00:00 2024-09-13T14:07:43.470 INFO:teuthology.orchestra.run.smithi145.stdout:(77/114): python3-babel-2.9.1-2.el9.noarch.rpm 2.6 MB/s | 6.0 MB 00:02 2024-09-13T14:07:43.520 INFO:teuthology.orchestra.run.smithi145.stdout:(78/114): liboath-2.6.7-2.el9.x86_64.rpm 976 kB/s | 49 kB 00:00 2024-09-13T14:07:43.522 INFO:teuthology.orchestra.run.smithi026.stdout:(66/114): python3-prettytable-0.7.2-27.el9.noar 727 kB/s | 42 kB 00:00 2024-09-13T14:07:43.554 INFO:teuthology.orchestra.run.smithi145.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.0 MB/s | 67 kB 00:00 2024-09-13T14:07:43.579 INFO:teuthology.orchestra.run.smithi145.stdout:(80/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 1.6 MB/s | 308 kB 00:00 2024-09-13T14:07:43.604 INFO:teuthology.orchestra.run.smithi145.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-13T14:07:43.629 INFO:teuthology.orchestra.run.smithi145.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.3 MB/s | 32 kB 00:00 2024-09-13T14:07:43.647 INFO:teuthology.orchestra.run.smithi026.stdout:(67/114): python3-py-1.10.0-6.el9.noarch.rpm 3.7 MB/s | 477 kB 00:00 2024-09-13T14:07:43.663 INFO:teuthology.orchestra.run.smithi145.stdout:(83/114): python3-asyncssh-2.13.2-5.el9.noarch. 4.9 MB/s | 548 kB 00:00 2024-09-13T14:07:43.688 INFO:teuthology.orchestra.run.smithi145.stdout:(84/114): python3-certifi-2023.05.07-4.el9.noar 241 kB/s | 14 kB 00:00 2024-09-13T14:07:43.714 INFO:teuthology.orchestra.run.smithi145.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 3.3 MB/s | 172 kB 00:00 2024-09-13T14:07:43.739 INFO:teuthology.orchestra.run.smithi145.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 7.0 MB/s | 358 kB 00:00 2024-09-13T14:07:43.748 INFO:teuthology.orchestra.run.smithi026.stdout:(68/114): python3-babel-2.9.1-2.el9.noarch.rpm 3.9 MB/s | 6.0 MB 00:01 2024-09-13T14:07:43.772 INFO:teuthology.orchestra.run.smithi145.stdout:(87/114): python3-google-auth-2.34.0-1.el9.noar 3.7 MB/s | 220 kB 00:00 2024-09-13T14:07:43.773 INFO:teuthology.orchestra.run.smithi026.stdout:(69/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-09-13T14:07:43.798 INFO:teuthology.orchestra.run.smithi145.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 183 kB/s | 11 kB 00:00 2024-09-13T14:07:43.823 INFO:teuthology.orchestra.run.smithi145.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 352 kB/s | 18 kB 00:00 2024-09-13T14:07:43.831 INFO:teuthology.orchestra.run.smithi026.stdout:(70/114): python3-pyasn1-modules-0.4.8-6.el9.no 3.3 MB/s | 279 kB 00:00 2024-09-13T14:07:43.848 INFO:teuthology.orchestra.run.smithi145.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 461 kB/s | 23 kB 00:00 2024-09-13T14:07:43.873 INFO:teuthology.orchestra.run.smithi145.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 388 kB/s | 19 kB 00:00 2024-09-13T14:07:43.898 INFO:teuthology.orchestra.run.smithi026.stdout:(71/114): python3-pytest-6.2.2-7.el9.noarch.rpm 4.0 MB/s | 519 kB 00:00 2024-09-13T14:07:43.899 INFO:teuthology.orchestra.run.smithi145.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 393 kB/s | 20 kB 00:00 2024-09-13T14:07:43.924 INFO:teuthology.orchestra.run.smithi026.stdout:(72/114): python3-pytz-2021.1-5.el9.noarch.rpm 555 kB/s | 51 kB 00:00 2024-09-13T14:07:43.924 INFO:teuthology.orchestra.run.smithi145.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 180 kB/s | 9.0 kB 00:00 2024-09-13T14:07:43.950 INFO:teuthology.orchestra.run.smithi145.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 810 kB/s | 41 kB 00:00 2024-09-13T14:07:43.975 INFO:teuthology.orchestra.run.smithi145.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-13T14:07:43.999 INFO:teuthology.orchestra.run.smithi026.stdout:(73/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.8 MB/s | 5.1 MB 00:01 2024-09-13T14:07:44.000 INFO:teuthology.orchestra.run.smithi145.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-13T14:07:44.024 INFO:teuthology.orchestra.run.smithi026.stdout:(74/114): python3-requests-oauthlib-1.3.0-12.el 428 kB/s | 54 kB 00:00 2024-09-13T14:07:44.034 INFO:teuthology.orchestra.run.smithi145.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 1.7 MB/s | 58 kB 00:00 2024-09-13T14:07:44.084 INFO:teuthology.orchestra.run.smithi145.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 5.3 MB/s | 272 kB 00:00 2024-09-13T14:07:44.108 INFO:teuthology.orchestra.run.smithi026.stdout:(75/114): socat-1.7.4.1-6.el9.x86_64.rpm 3.6 MB/s | 304 kB 00:00 2024-09-13T14:07:44.117 INFO:teuthology.orchestra.run.smithi145.stdout:(99/114): python3-kubernetes-26.1.0-2.el9.noarc 5.3 MB/s | 1.0 MB 00:00 2024-09-13T14:07:44.133 INFO:teuthology.orchestra.run.smithi026.stdout:(76/114): python3-toml-0.10.2-6.el9.noarch.rpm 311 kB/s | 42 kB 00:00 2024-09-13T14:07:44.143 INFO:teuthology.orchestra.run.smithi145.stdout:(100/114): python3-portend-3.1.0-2.el9.noarch.r 280 kB/s | 16 kB 00:00 2024-09-13T14:07:44.167 INFO:teuthology.orchestra.run.smithi026.stdout:(77/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-09-13T14:07:44.168 INFO:teuthology.orchestra.run.smithi145.stdout:(101/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.8 MB/s | 90 kB 00:00 2024-09-13T14:07:44.193 INFO:teuthology.orchestra.run.smithi145.stdout:(102/114): python3-repoze-lru-0.7-16.el9.noarch 614 kB/s | 31 kB 00:00 2024-09-13T14:07:44.217 INFO:teuthology.orchestra.run.smithi026.stdout:(78/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 3.6 MB/s | 308 kB 00:00 2024-09-13T14:07:44.218 INFO:teuthology.orchestra.run.smithi145.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-13T14:07:44.242 INFO:teuthology.orchestra.run.smithi026.stdout:(79/114): liboath-2.6.7-2.el9.x86_64.rpm 665 kB/s | 49 kB 00:00 2024-09-13T14:07:44.244 INFO:teuthology.orchestra.run.smithi145.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 2.4 MB/s | 188 kB 00:00 2024-09-13T14:07:44.267 INFO:teuthology.orchestra.run.smithi026.stdout:(80/114): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-13T14:07:44.269 INFO:teuthology.orchestra.run.smithi145.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 712 kB/s | 36 kB 00:00 2024-09-13T14:07:44.293 INFO:teuthology.orchestra.run.smithi026.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-13T14:07:44.295 INFO:teuthology.orchestra.run.smithi145.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.5 MB/s | 74 kB 00:00 2024-09-13T14:07:44.318 INFO:teuthology.orchestra.run.smithi026.stdout:(82/114): python3-asyncssh-2.13.2-5.el9.noarch. 7.1 MB/s | 548 kB 00:00 2024-09-13T14:07:44.320 INFO:teuthology.orchestra.run.smithi145.stdout:(107/114): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-13T14:07:44.344 INFO:teuthology.orchestra.run.smithi026.stdout:(83/114): python3-cachetools-4.2.4-1.el9.noarch 642 kB/s | 32 kB 00:00 2024-09-13T14:07:44.345 INFO:teuthology.orchestra.run.smithi145.stdout:(108/114): python3-websocket-client-1.2.3-2.el9 1.7 MB/s | 90 kB 00:00 2024-09-13T14:07:44.369 INFO:teuthology.orchestra.run.smithi026.stdout:(84/114): python3-certifi-2023.05.07-4.el9.noar 281 kB/s | 14 kB 00:00 2024-09-13T14:07:44.370 INFO:teuthology.orchestra.run.smithi145.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 804 kB/s | 20 kB 00:00 2024-09-13T14:07:44.394 INFO:teuthology.orchestra.run.smithi026.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 3.4 MB/s | 172 kB 00:00 2024-09-13T14:07:44.419 INFO:teuthology.orchestra.run.smithi026.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 7.0 MB/s | 358 kB 00:00 2024-09-13T14:07:44.421 INFO:teuthology.orchestra.run.smithi145.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 4.1 MB/s | 427 kB 00:00 2024-09-13T14:07:44.453 INFO:teuthology.orchestra.run.smithi026.stdout:(87/114): python3-google-auth-2.34.0-1.el9.noar 3.7 MB/s | 220 kB 00:00 2024-09-13T14:07:44.478 INFO:teuthology.orchestra.run.smithi026.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 182 kB/s | 11 kB 00:00 2024-09-13T14:07:44.503 INFO:teuthology.orchestra.run.smithi026.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 352 kB/s | 18 kB 00:00 2024-09-13T14:07:44.529 INFO:teuthology.orchestra.run.smithi026.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 460 kB/s | 23 kB 00:00 2024-09-13T14:07:44.529 INFO:teuthology.orchestra.run.smithi145.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 10 MB/s | 1.6 MB 00:00 2024-09-13T14:07:44.554 INFO:teuthology.orchestra.run.smithi026.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 387 kB/s | 19 kB 00:00 2024-09-13T14:07:44.579 INFO:teuthology.orchestra.run.smithi026.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 394 kB/s | 20 kB 00:00 2024-09-13T14:07:44.604 INFO:teuthology.orchestra.run.smithi026.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 181 kB/s | 9.0 kB 00:00 2024-09-13T14:07:44.630 INFO:teuthology.orchestra.run.smithi026.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 810 kB/s | 41 kB 00:00 2024-09-13T14:07:44.655 INFO:teuthology.orchestra.run.smithi026.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-13T14:07:44.663 INFO:teuthology.orchestra.run.smithi145.stdout:(112/114): librados2-17.2.7-1627.gbbdadfa3.el9. 13 MB/s | 3.1 MB 00:00 2024-09-13T14:07:44.681 INFO:teuthology.orchestra.run.smithi026.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-13T14:07:44.706 INFO:teuthology.orchestra.run.smithi026.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-13T14:07:44.748 INFO:teuthology.orchestra.run.smithi026.stdout:(98/114): python3-kubernetes-26.1.0-2.el9.noarc 7.1 MB/s | 1.0 MB 00:00 2024-09-13T14:07:44.781 INFO:teuthology.orchestra.run.smithi026.stdout:(99/114): python3-pecan-1.4.2-3.el9.noarch.rpm 3.5 MB/s | 272 kB 00:00 2024-09-13T14:07:44.807 INFO:teuthology.orchestra.run.smithi026.stdout:(100/114): python3-portend-3.1.0-2.el9.noarch.r 280 kB/s | 16 kB 00:00 2024-09-13T14:07:44.832 INFO:teuthology.orchestra.run.smithi026.stdout:(101/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.7 MB/s | 90 kB 00:00 2024-09-13T14:07:44.838 INFO:teuthology.orchestra.run.smithi145.stdout:(113/114): librbd1-17.2.7-1627.gbbdadfa3.el9.x8 9.5 MB/s | 2.9 MB 00:00 2024-09-13T14:07:44.857 INFO:teuthology.orchestra.run.smithi026.stdout:(102/114): python3-repoze-lru-0.7-16.el9.noarch 612 kB/s | 31 kB 00:00 2024-09-13T14:07:44.883 INFO:teuthology.orchestra.run.smithi026.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-13T14:07:44.908 INFO:teuthology.orchestra.run.smithi026.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 2.4 MB/s | 188 kB 00:00 2024-09-13T14:07:44.933 INFO:teuthology.orchestra.run.smithi026.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 714 kB/s | 36 kB 00:00 2024-09-13T14:07:44.959 INFO:teuthology.orchestra.run.smithi026.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.5 MB/s | 74 kB 00:00 2024-09-13T14:07:44.984 INFO:teuthology.orchestra.run.smithi026.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 3.5 MB/s | 90 kB 00:00 2024-09-13T14:07:45.009 INFO:teuthology.orchestra.run.smithi026.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-13T14:07:45.034 INFO:teuthology.orchestra.run.smithi026.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 803 kB/s | 20 kB 00:00 2024-09-13T14:07:45.060 INFO:teuthology.orchestra.run.smithi026.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 5.5 MB/s | 427 kB 00:00 2024-09-13T14:07:45.163 INFO:teuthology.orchestra.run.smithi145.stdout:(114/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 6.6 MB/s | 16 MB 00:02 2024-09-13T14:07:45.168 INFO:teuthology.orchestra.run.smithi145.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:07:45.169 INFO:teuthology.orchestra.run.smithi145.stdout:Total 18 MB/s | 190 MB 00:10 2024-09-13T14:07:45.235 INFO:teuthology.orchestra.run.smithi026.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 7.9 MB/s | 1.6 MB 00:00 2024-09-13T14:07:45.510 INFO:teuthology.orchestra.run.smithi026.stdout:(112/114): librados2-17.2.7-1627.gbbdadfa3.el9. 6.8 MB/s | 3.1 MB 00:00 2024-09-13T14:07:45.594 INFO:teuthology.orchestra.run.smithi026.stdout:(113/114): librbd1-17.2.7-1627.gbbdadfa3.el9.x8 8.2 MB/s | 2.9 MB 00:00 2024-09-13T14:07:45.920 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check 2024-09-13T14:07:46.014 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded. 2024-09-13T14:07:46.015 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test 2024-09-13T14:07:46.177 INFO:teuthology.orchestra.run.smithi026.stdout:(114/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 7.2 MB/s | 16 MB 00:02 2024-09-13T14:07:46.183 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:07:46.183 INFO:teuthology.orchestra.run.smithi026.stdout:Total 17 MB/s | 190 MB 00:11 2024-09-13T14:07:46.957 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-09-13T14:07:47.062 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-09-13T14:07:47.062 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-09-13T14:07:47.800 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded. 2024-09-13T14:07:47.801 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction 2024-09-13T14:07:48.804 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-09-13T14:07:48.805 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-09-13T14:07:52.511 INFO:teuthology.orchestra.run.smithi145.stdout: Preparing : 1/1 2024-09-13T14:07:52.574 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-13T14:07:52.599 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-13T14:07:52.654 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-13T14:07:52.970 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-13T14:07:53.100 INFO:teuthology.orchestra.run.smithi145.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/116 2024-09-13T14:07:53.206 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/116 2024-09-13T14:07:53.235 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:07:53.332 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:07:53.384 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/116 2024-09-13T14:07:53.414 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-13T14:07:53.759 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-09-13T14:07:53.839 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-13T14:07:53.848 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-13T14:07:53.872 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-13T14:07:53.879 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:07:53.936 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-13T14:07:53.953 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:07:53.980 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:07:54.080 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:07:54.105 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 12/116 2024-09-13T14:07:54.232 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 12/116 2024-09-13T14:07:54.267 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-13T14:07:54.268 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-13T14:07:54.308 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-13T14:07:54.338 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-13T14:07:54.373 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/116 2024-09-13T14:07:54.379 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-13T14:07:54.444 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-13T14:07:54.489 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-13T14:07:54.489 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/116 2024-09-13T14:07:54.515 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:07:54.520 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-13T14:07:54.547 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-13T14:07:54.617 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-13T14:07:54.622 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:07:54.669 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-13T14:07:54.679 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/116 2024-09-13T14:07:54.705 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-13T14:07:54.712 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-13T14:07:54.744 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-13T14:07:54.779 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-13T14:07:54.822 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el 26/116 2024-09-13T14:07:54.898 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_6 27/116 2024-09-13T14:07:55.038 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-13T14:07:55.091 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-13T14:07:55.174 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-13T14:07:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:07:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:07:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:07:55.371 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:07:55.395 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 12/116 2024-09-13T14:07:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 12/116 2024-09-13T14:07:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-13T14:07:55.607 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-13T14:07:55.616 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-13T14:07:55.637 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-13T14:07:55.678 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-13T14:07:55.696 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-13T14:07:55.758 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-13T14:07:55.762 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-13T14:07:55.792 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-13T14:07:55.821 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-13T14:07:55.826 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-13T14:07:55.861 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-13T14:07:55.865 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-13T14:07:55.888 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-13T14:07:55.901 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 36/116 2024-09-13T14:07:55.940 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_6 37/116 2024-09-13T14:07:55.960 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-13T14:07:55.970 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-13T14:07:56.006 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-13T14:07:56.012 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-13T14:07:56.044 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-13T14:07:56.062 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-13T14:07:56.085 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-13T14:07:56.128 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-13T14:07:56.133 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-13T14:07:56.166 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-13T14:07:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el 26/116 2024-09-13T14:07:56.202 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-13T14:07:56.249 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-13T14:07:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_6 27/116 2024-09-13T14:07:56.302 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-13T14:07:56.344 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-13T14:07:56.421 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-13T14:07:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-13T14:07:56.481 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-13T14:07:56.530 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-13T14:07:56.571 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-13T14:07:56.674 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-13T14:07:56.722 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-13T14:07:56.762 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-13T14:07:56.799 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-13T14:07:56.838 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-13T14:07:56.962 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-13T14:07:56.996 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-13T14:07:57.036 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-13T14:07:57.043 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-13T14:07:57.131 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-13T14:07:57.132 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-13T14:07:57.174 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-13T14:07:57.189 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-13T14:07:57.207 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-13T14:07:57.242 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-13T14:07:57.281 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-13T14:07:57.317 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 36/116 2024-09-13T14:07:57.364 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_6 37/116 2024-09-13T14:07:57.394 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-13T14:07:57.431 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-13T14:07:57.488 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-13T14:07:57.562 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-13T14:07:57.602 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-13T14:07:57.638 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-13T14:07:57.686 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-13T14:07:57.731 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-13T14:07:57.773 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-13T14:07:57.912 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-13T14:07:57.918 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-13T14:07:57.953 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-13T14:07:57.957 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-13T14:07:57.986 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-13T14:07:58.008 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-13T14:07:58.020 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-13T14:07:58.120 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-13T14:07:58.125 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-13T14:07:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-13T14:07:58.178 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-13T14:07:58.206 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-13T14:07:58.222 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-13T14:07:58.243 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-13T14:07:58.288 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-13T14:07:58.404 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-13T14:07:58.437 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-13T14:07:58.437 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-13T14:07:58.479 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-13T14:07:58.513 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-13T14:07:58.519 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-13T14:07:58.632 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-13T14:07:58.682 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-13T14:07:59.169 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-13T14:07:59.293 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-13T14:07:59.461 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-13T14:07:59.511 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-13T14:07:59.553 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-13T14:07:59.589 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-13T14:07:59.692 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-13T14:07:59.746 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-13T14:07:59.780 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-13T14:07:59.972 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-13T14:08:00.012 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-13T14:08:00.095 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-13T14:08:00.549 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-13T14:08:00.642 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-13T14:08:00.810 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-13T14:08:00.933 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-13T14:08:00.939 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-13T14:08:00.971 INFO:teuthology.orchestra.run.smithi145.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 75/116 2024-09-13T14:08:01.061 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 75/116 2024-09-13T14:08:01.100 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 76/116 2024-09-13T14:08:01.149 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-13T14:08:01.205 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-13T14:08:01.399 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-13T14:08:01.446 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-13T14:08:01.525 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-13T14:08:01.614 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-13T14:08:01.755 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-13T14:08:01.801 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-13T14:08:01.830 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-13T14:08:01.871 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-13T14:08:01.903 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-13T14:08:01.940 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-13T14:08:02.327 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-13T14:08:02.401 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 89/116 2024-09-13T14:08:02.417 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-13T14:08:02.428 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 89/116 2024-09-13T14:08:02.460 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.e 90/116 2024-09-13T14:08:02.549 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3. 91/116 2024-09-13T14:08:02.579 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9. 92/116 2024-09-13T14:08:02.725 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-13T14:08:02.770 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 75/116 2024-09-13T14:08:02.861 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 75/116 2024-09-13T14:08:02.901 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 76/116 2024-09-13T14:08:02.959 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-13T14:08:03.023 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-13T14:08:03.229 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-13T14:08:03.280 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-13T14:08:03.363 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-13T14:08:03.459 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-13T14:08:03.604 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-13T14:08:03.659 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-13T14:08:03.688 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-13T14:08:03.722 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-13T14:08:03.754 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-13T14:08:03.781 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-13T14:08:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 89/116 2024-09-13T14:08:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 89/116 2024-09-13T14:08:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.e 90/116 2024-09-13T14:08:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3. 91/116 2024-09-13T14:08:04.414 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9. 92/116 2024-09-13T14:08:05.016 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:05.095 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:05.630 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:05.664 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 94/116 2024-09-13T14:08:05.761 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 94/116 2024-09-13T14:08:05.761 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:08:05.761 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:08:05.762 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:05.799 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 95/116 2024-09-13T14:08:07.197 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:07.235 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:07.776 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 93/116 2024-09-13T14:08:07.822 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 94/116 2024-09-13T14:08:07.928 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 94/116 2024-09-13T14:08:07.928 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:08:07.928 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:08:07.928 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:07.965 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 95/116 2024-09-13T14:08:17.272 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 95/116 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /sys 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /proc 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /mnt 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /var/tmp 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /home 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /root 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout:skipping the directory /tmp 2024-09-13T14:08:17.273 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:17.341 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 96/116 2024-09-13T14:08:18.351 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 96/116 2024-09-13T14:08:18.404 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 97/116 2024-09-13T14:08:19.361 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 97/116 2024-09-13T14:08:19.387 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 98/116 2024-09-13T14:08:19.489 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 98/116 2024-09-13T14:08:19.641 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el 99/116 2024-09-13T14:08:19.664 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 100/116 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 100/116 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:19.688 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:19.731 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 101/116 2024-09-13T14:08:19.870 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 95/116 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /sys 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /proc 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /mnt 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /var/tmp 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /home 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /root 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /tmp 2024-09-13T14:08:19.871 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:19.914 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 101/116 2024-09-13T14:08:19.940 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 102/116 2024-09-13T14:08:19.942 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 96/116 2024-09-13T14:08:19.964 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 102/116 2024-09-13T14:08:19.964 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:19.964 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:08:19.964 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:19.964 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:19.965 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:20.332 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 103/116 2024-09-13T14:08:20.356 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 103/116 2024-09-13T14:08:20.356 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:20.356 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:08:20.356 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:20.356 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:20.357 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:20.987 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 96/116 2024-09-13T14:08:21.030 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 97/116 2024-09-13T14:08:21.666 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 104/116 2024-09-13T14:08:21.693 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 104/116 2024-09-13T14:08:21.693 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:21.693 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:08:21.693 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:21.694 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:21.694 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:22.044 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 97/116 2024-09-13T14:08:22.071 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 98/116 2024-09-13T14:08:22.180 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 98/116 2024-09-13T14:08:22.349 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el 99/116 2024-09-13T14:08:22.382 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 100/116 2024-09-13T14:08:22.407 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 100/116 2024-09-13T14:08:22.407 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:22.407 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:08:22.408 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:22.408 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:22.408 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:22.449 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 101/116 2024-09-13T14:08:22.642 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 101/116 2024-09-13T14:08:22.675 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 102/116 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 102/116 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:22.700 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:22.940 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 105/116 2024-09-13T14:08:22.977 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 106/116 2024-09-13T14:08:23.054 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 106/116 2024-09-13T14:08:23.055 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:23.055 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:08:23.055 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:23.055 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:23.055 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:23.103 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 107/116 2024-09-13T14:08:23.109 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 103/116 2024-09-13T14:08:23.126 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 107/116 2024-09-13T14:08:23.126 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:23.126 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:08:23.127 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:23.134 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 103/116 2024-09-13T14:08:23.134 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:23.134 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:08:23.134 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:23.134 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:23.135 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:23.420 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 108/116 2024-09-13T14:08:23.443 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 108/116 2024-09-13T14:08:23.443 INFO:teuthology.orchestra.run.smithi145.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:23.443 INFO:teuthology.orchestra.run.smithi145.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:08:23.443 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:08:23.444 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:08:23.444 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 104/116 2024-09-13T14:08:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 104/116 2024-09-13T14:08:24.539 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:24.539 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:08:24.539 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:24.539 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:25.875 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 105/116 2024-09-13T14:08:25.903 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 106/116 2024-09-13T14:08:25.982 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 106/116 2024-09-13T14:08:25.982 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:25.983 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:08:25.983 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:25.983 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:25.983 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:26.020 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 107/116 2024-09-13T14:08:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 107/116 2024-09-13T14:08:26.044 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:26.044 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:08:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:26.354 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 108/116 2024-09-13T14:08:26.379 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 108/116 2024-09-13T14:08:26.379 INFO:teuthology.orchestra.run.smithi026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:08:26.379 INFO:teuthology.orchestra.run.smithi026.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:08:26.380 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:08:26.380 INFO:teuthology.orchestra.run.smithi026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:08:26.380 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:27.866 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 109/116 2024-09-13T14:08:27.897 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 110/116 2024-09-13T14:08:27.939 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 111/116 2024-09-13T14:08:28.057 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 112/116 2024-09-13T14:08:28.104 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-13T14:08:28.128 INFO:teuthology.orchestra.run.smithi145.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_ 114/116 2024-09-13T14:08:28.128 INFO:teuthology.orchestra.run.smithi145.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-13T14:08:28.220 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-13T14:08:28.220 INFO:teuthology.orchestra.run.smithi145.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 109/116 2024-09-13T14:08:31.117 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 110/116 2024-09-13T14:08:31.158 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 111/116 2024-09-13T14:08:31.285 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 112/116 2024-09-13T14:08:31.323 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-13T14:08:31.346 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_ 114/116 2024-09-13T14:08:31.347 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-13T14:08:31.413 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-13T14:08:31.414 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 5/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:08:33.360 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 9/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_ 13/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 14/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 15/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_6 16/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 17/116 2024-09-13T14:08:33.361 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el 19/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9. 20/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_6 21/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 26/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 27/116 2024-09-13T14:08:33.362 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3. 28/116 2024-09-13T14:08:33.371 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 29/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 30/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 31/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el 32/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 33/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.e 34/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 35/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-13T14:08:33.372 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-13T14:08:33.373 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-13T14:08:33.374 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-13T14:08:33.375 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-13T14:08:33.376 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-13T14:08:33.377 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-13T14:08:33.378 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-13T14:08:33.379 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 113/116 2024-09-13T14:08:33.379 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-13T14:08:33.379 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 115/116 2024-09-13T14:08:35.368 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout:Upgraded: 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout:Installed: 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.369 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.370 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.371 INFO:teuthology.orchestra.run.smithi145.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.372 INFO:teuthology.orchestra.run.smithi145.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.373 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:08:35.374 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:08:35.375 INFO:teuthology.orchestra.run.smithi145.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:08:35.376 INFO:teuthology.orchestra.run.smithi145.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:08:35.377 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:08:35.378 INFO:teuthology.orchestra.run.smithi145.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:35.379 INFO:teuthology.orchestra.run.smithi145.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:08:35.380 INFO:teuthology.orchestra.run.smithi145.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:08:35.380 INFO:teuthology.orchestra.run.smithi145.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:08:35.380 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:08:35.380 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-09-13T14:08:35.684 DEBUG:teuthology.parallel:result is None 2024-09-13T14:08:36.445 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:36.445 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/116 2024-09-13T14:08:36.445 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/116 2024-09-13T14:08:36.445 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdad 5/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 6/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 9/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_ 13/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 14/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 15/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_6 16/116 2024-09-13T14:08:36.446 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86 17/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el 19/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9. 20/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_6 21/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 26/116 2024-09-13T14:08:36.447 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 27/116 2024-09-13T14:08:36.455 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3. 28/116 2024-09-13T14:08:36.455 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noa 29/116 2024-09-13T14:08:36.455 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.n 30/116 2024-09-13T14:08:36.455 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbd 31/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el 32/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 33/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.e 34/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 35/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-13T14:08:36.456 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-13T14:08:36.457 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-13T14:08:36.458 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-13T14:08:36.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-13T14:08:36.460 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-13T14:08:36.461 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 113/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-13T14:08:36.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 115/116 2024-09-13T14:08:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-13T14:08:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout:Upgraded: 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:08:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:08:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:08:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:08:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:08:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:08:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:08:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:08:38.441 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-09-13T14:08:38.742 DEBUG:teuthology.parallel:result is None 2024-09-13T14:08:38.743 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:08:38.743 INFO:teuthology.packaging:ref: None 2024-09-13T14:08:38.743 INFO:teuthology.packaging:tag: None 2024-09-13T14:08:38.743 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:08:38.743 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:08:38.743 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:08:38.974 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:08:38.997 INFO:teuthology.orchestra.run.smithi026.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:08:38.997 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:08:38.998 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:08:38.999 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:08:38.999 INFO:teuthology.packaging:ref: None 2024-09-13T14:08:39.000 INFO:teuthology.packaging:tag: None 2024-09-13T14:08:39.000 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:08:39.000 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:08:39.000 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:08:39.220 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:08:39.247 INFO:teuthology.orchestra.run.smithi145.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:08:39.247 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:08:39.247 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:08:39.249 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T14:08:39.249 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:08:39.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:08:39.277 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:08:39.277 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:08:39.320 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T14:08:39.320 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:08:39.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:08:39.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:08:39.413 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:08:39.413 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:08:39.443 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:08:39.512 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T14:08:39.512 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:08:39.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:08:39.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:08:39.603 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:08:39.603 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:08:39.632 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:08:39.699 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T14:08:39.700 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:08:39.700 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:08:39.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:08:39.791 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:08:39.791 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:08:39.817 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:08:39.884 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:08:39.893 INFO:teuthology.task.print:**** done install task... 2024-09-13T14:08:39.893 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:08:39.927 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-13T14:08:39.927 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T14:08:40.061 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:08:40.061 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:08:40.062 INFO:tasks.cephadm:Cluster fsid is ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:40.062 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T14:08:40.062 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.26', 'mon.c': '[v2:172.21.15.26:3301,v1:172.21.15.26:6790]', 'mon.b': '172.21.15.145'} 2024-09-13T14:08:40.062 INFO:tasks.cephadm:First mon is mon.a on smithi026 2024-09-13T14:08:40.062 INFO:tasks.cephadm:First mgr is y 2024-09-13T14:08:40.062 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T14:08:40.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo hostname $(hostname -s) 2024-09-13T14:08:40.089 DEBUG:teuthology.orchestra.run.smithi145:> sudo hostname $(hostname -s) 2024-09-13T14:08:40.116 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-13T14:08:40.117 DEBUG:teuthology.orchestra.run.smithi026:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.262 INFO:teuthology.orchestra.run.smithi026.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:08 /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.263 DEBUG:teuthology.orchestra.run.smithi145:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.523 INFO:teuthology.orchestra.run.smithi145.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:08 /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.523 DEBUG:teuthology.orchestra.run.smithi026:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.542 DEBUG:teuthology.orchestra.run.smithi145:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:40.573 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-13T14:08:40.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:08:40.585 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:08:42.220 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:08:42.268 INFO:teuthology.orchestra.run.smithi145.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:09:43.663 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "repo_digests": [ 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: ] 2024-09-13T14:09:43.664 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-09-13T14:09:43.677 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "repo_digests": [ 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-09-13T14:09:43.678 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:09:43.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo mkdir -p /etc/ceph 2024-09-13T14:09:43.746 DEBUG:teuthology.orchestra.run.smithi145:> sudo mkdir -p /etc/ceph 2024-09-13T14:09:43.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 777 /etc/ceph 2024-09-13T14:09:43.801 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 777 /etc/ceph 2024-09-13T14:09:43.841 INFO:tasks.cephadm:Writing seed config... 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T14:09:43.843 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T14:09:43.844 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:09:43.844 DEBUG:teuthology.orchestra.run.smithi026:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T14:09:43.860 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ad455616-71d9-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T14:09:43.860 DEBUG:teuthology.orchestra.run.smithi026:mon.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T14:09:43.902 DEBUG:teuthology.orchestra.run.smithi026:mgr.y> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T14:09:43.945 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T14:09:43.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid ad455616-71d9-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.26 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:09:44.165 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:09:44.165 INFO:teuthology.orchestra.run.smithi026.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'ad455616-71d9-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.26', '--skip-admin-label'] 2024-09-13T14:09:44.190 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:09:44.190 INFO:teuthology.orchestra.run.smithi026.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T14:09:44.191 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying podman|docker is present... 2024-09-13T14:09:44.214 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:09:44.214 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying lvm2 is present... 2024-09-13T14:09:44.214 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying time synchronization is in place... 2024-09-13T14:09:44.222 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:09:44.222 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:09:44.229 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:09:44.229 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:09:44.236 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2024-09-13T14:09:44.243 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2024-09-13T14:09:44.243 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:09:44.244 INFO:teuthology.orchestra.run.smithi026.stdout:Repeating the final host check... 2024-09-13T14:09:44.266 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:09:44.266 INFO:teuthology.orchestra.run.smithi026.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-13T14:09:44.266 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl is present 2024-09-13T14:09:44.266 INFO:teuthology.orchestra.run.smithi026.stdout:lvcreate is present 2024-09-13T14:09:44.273 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:09:44.273 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:09:44.280 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:09:44.280 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:09:44.287 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2024-09-13T14:09:44.294 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2024-09-13T14:09:44.294 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:09:44.294 INFO:teuthology.orchestra.run.smithi026.stdout:Host looks OK 2024-09-13T14:09:44.294 INFO:teuthology.orchestra.run.smithi026.stdout:Cluster fsid: ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:44.294 INFO:teuthology.orchestra.run.smithi026.stdout:Acquiring lock 140640705573792 on /run/cephadm/ad455616-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:09:44.295 INFO:teuthology.orchestra.run.smithi026.stdout:Lock 140640705573792 acquired on /run/cephadm/ad455616-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:09:44.295 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 3300 ... 2024-09-13T14:09:44.295 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 6789 ... 2024-09-13T14:09:44.295 INFO:teuthology.orchestra.run.smithi026.stdout:Base mon IP(s) is [172.21.15.26:3300, 172.21.15.26:6789], mon addrv is [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-09-13T14:09:44.299 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.26 metric 100 2024-09-13T14:09:44.299 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.26 metric 100 2024-09-13T14:09:44.302 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T14:09:44.302 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T14:09:44.302 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1794sec hoplimit 64 pref medium 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:cac0/64 scope link 2024-09-13T14:09:44.305 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:09:44.306 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2024-09-13T14:09:44.306 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2024-09-13T14:09:44.306 INFO:teuthology.orchestra.run.smithi026.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-13T14:09:44.306 INFO:teuthology.orchestra.run.smithi026.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T14:09:44.307 INFO:teuthology.orchestra.run.smithi026.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Getting image source signatures 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:ca308b3f556da579f99cd79db684ebbf678db4c29a649b66aaf6f99bc8124cd6 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying config sha256:a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:09:44.811 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-13T14:09:45.957 INFO:teuthology.orchestra.run.smithi026.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:09:45.957 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:09:45.957 INFO:teuthology.orchestra.run.smithi026.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T14:09:46.970 INFO:teuthology.orchestra.run.smithi026.stdout:stat: stdout 167 167 2024-09-13T14:09:46.971 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial keys... 2024-09-13T14:09:48.159 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQCrR+Rm9q6TGhAAa+XQM1bSCkEVscXEAg2M8Q== 2024-09-13T14:09:49.290 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQCsR+RmiK04IBAAT+db+KXI7CaeNzCigT65Tg== 2024-09-13T14:09:50.412 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQCtR+RmEHYWLxAAaheUZgLx7IJZES73xrwUTw== 2024-09-13T14:09:50.412 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial monmap... 2024-09-13T14:09:51.538 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:09:51.538 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-13T14:09:51.538 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:51.538 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout:monmaptool for a [v2:172.21.15.26:3300,v1:172.21.15.26:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout:setting min_mon_release = octopus 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: set fsid to ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:09:51.539 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mon... 2024-09-13T14:09:52.492 INFO:teuthology.orchestra.run.smithi026.stdout:create mon.a on 2024-09-13T14:09:52.708 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-13T14:09:52.856 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T14:09:53.013 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:09:53.013 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:09:53.186 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:09:53.186 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T14:09:53.356 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target.wants/ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:09:53.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:53 smithi026 ceph-mon[27297]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:09:53.857 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:09:53.857 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2024-09-13T14:09:53.864 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:09:53.864 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:09:53.864 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2024-09-13T14:09:53.865 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:09:53.865 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon to start... 2024-09-13T14:09:53.865 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon... 2024-09-13T14:09:54.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:53 smithi026 bash[27259]: 11eb77dca6ce2b0e6de0c3152d92269bda4b5ea52246caf1500d4a2f0d4fba95 2024-09-13T14:09:54.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:53 smithi026 systemd[1]: Started Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:09:54.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:09:54.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:09:54.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: fsmap 2024-09-13T14:09:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:09:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: mgrmap e1: no daemons active 2024-09-13T14:09:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:54 smithi026 ceph-mon[27297]: from='client.? 172.21.15.26:0/516440086' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout id: ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout services: 2024-09-13T14:09:54.970 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.571507s) 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout data: 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:mon is available 2024-09-13T14:09:54.971 INFO:teuthology.orchestra.run.smithi026.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T14:09:55.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:55 smithi026 ceph-mon[27297]: from='client.? 172.21.15.26:0/3015173853' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:09:55.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:55 smithi026 ceph-mon[27297]: from='client.? 172.21.15.26:0/3015173853' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:09:55.959 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:09:55.960 INFO:teuthology.orchestra.run.smithi026.stdout:Generating new minimal ceph.conf... 2024-09-13T14:09:57.006 INFO:teuthology.orchestra.run.smithi026.stdout:Restarting the monitor... 2024-09-13T14:09:57.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:56 smithi026 ceph-mon[27297]: from='client.? 172.21.15.26:0/3812500981' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:57.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 systemd[1]: Stopping Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:09:57.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[27274]: 2024-09-13T14:09:57.281+0000 7f14045e6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:09:57.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[27274]: 2024-09-13T14:09:57.281+0000 7f14045e6640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T14:09:57.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 podman[27614]: 2024-09-13 14:09:57.487773193 +0000 UTC m=+0.309707482 container died 11eb77dca6ce2b0e6de0c3152d92269bda4b5ea52246caf1500d4a2f0d4fba95 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa) 2024-09-13T14:09:58.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 podman[27614]: 2024-09-13 14:09:57.821491736 +0000 UTC m=+0.643426017 container cleanup 11eb77dca6ce2b0e6de0c3152d92269bda4b5ea52246caf1500d4a2f0d4fba95 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T14:09:58.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 bash[27614]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:09:58.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:57 smithi026 podman[27630]: 2024-09-13 14:09:57.989011439 +0000 UTC m=+0.500018827 container remove 11eb77dca6ce2b0e6de0c3152d92269bda4b5ea52246caf1500d4a2f0d4fba95 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS) 2024-09-13T14:09:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:09:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 systemd[1]: Stopped Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:09:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 systemd[1]: Starting Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:09:58.671 INFO:teuthology.orchestra.run.smithi026.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 podman[27732]: 2024-09-13 14:09:58.439847736 +0000 UTC m=+0.106958745 container create 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 podman[27732]: 2024-09-13 14:09:58.349051507 +0000 UTC m=+0.016162489 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 podman[27732]: 2024-09-13 14:09:58.600208803 +0000 UTC m=+0.267319773 container init 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 podman[27732]: 2024-09-13 14:09:58.605328048 +0000 UTC m=+0.272439016 container start 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: pidfile_write: ignore empty --pid-file 2024-09-13T14:09:58.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: load: jerasure load: lrc 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: DB SUMMARY 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: DB Session ID: IHN187QVL52GUSUTSN9N 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83684 ; 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:09:58.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.env: 0x55b96b491440 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.fs: Posix File System 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.info_log: 0x55b96daa4ec0 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:09:58.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.statistics: (nil) 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.use_fsync: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.db_log_dir: 2024-09-13T14:09:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.write_buffer_manager: 0x55b96db627e0 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:09:58.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.unordered_write: 0 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.row_cache: None 2024-09-13T14:09:58.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.wal_filter: None 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:09:58.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:09:58.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_open_files: -1 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Compression algorithms supported: 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kZSTD supported: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:09:58.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.merge_operator: 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_filter: None 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:09:58.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b96da4ba38) 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: hash_index_allow_collision: 1 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 1 2024-09-13T14:09:58.806 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x55b96da811f0 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-09-13T14:09:58.807 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 4 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression: NoCompression 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:09:58.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.num_levels: 7 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:09:58.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:09:58.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:09:58.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:09:58.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.ttl: 2592000 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:09:58.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236598629531, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236598630339, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80257, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78499, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72712, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236598, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ebdadac2-46c1-40d4-a3dd-15131196942b", "db_session_id": "IHN187QVL52GUSUTSN9N"}} 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-13T14:09:58.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236598630701, "job": 1, "event": "recovery_finished"} 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55b96dab4700 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: DB pointer 0x55b96da84000 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:09:58.815 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:09:58.816 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 23.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 23.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:09:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 143.3 0.00 0.00 1 0.001 0 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:09:58.818 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 23.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???) e1 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).mds e1 new map 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).mds e1 print_map 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:09:58.819 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 bash[27732]: 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 systemd[1]: Started Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: fsmap 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:09:58.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:09:58 smithi026 ceph-mon[27770]: mgrmap e1: no daemons active 2024-09-13T14:09:59.768 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T14:09:59.768 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:09:59.768 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mgr... 2024-09-13T14:09:59.769 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T14:09:59.769 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T14:09:59.952 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:09:59.952 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T14:10:00.106 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968.target.wants/ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-ad455616-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:10:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:00 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1051075608' entity='client.admin' 2024-09-13T14:10:00.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 podman[28006]: 2024-09-13 14:10:00.349352818 +0000 UTC m=+0.113145577 container create 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T14:10:00.666 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:10:00.666 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2024-09-13T14:10:00.674 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:10:00.674 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:10:00.674 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2024-09-13T14:10:00.674 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:10:00.683 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:10:00.683 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2024-09-13T14:10:00.691 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:10:00.692 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:10:00.692 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2024-09-13T14:10:00.692 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T14:10:00.692 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr to start... 2024-09-13T14:10:00.692 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr... 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 podman[28006]: 2024-09-13 14:10:00.543622427 +0000 UTC m=+0.307415184 container init 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 podman[28006]: 2024-09-13 14:10:00.548690501 +0000 UTC m=+0.312483258 container start 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 bash[28006]: 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 systemd[1]: Started Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:00.665+0000 7ff611786ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:10:00.794 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:00.725+0000 7ff611786ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:00.795 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:00.792+0000 7ff611786ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:10:01.070 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:00.922+0000 7ff611786ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:01.070 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:01.068+0000 7ff611786ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:10:01.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:01 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/851389400' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:01.346+0000 7ff611786ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:10:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:01.409+0000 7ff611786ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:10:01.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:01.579+0000 7ff611786ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:10:02.329 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:02.332 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:02.332 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "ad455616-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:02.332 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:02.332 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:02.333 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:02.334 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:02.337 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.337 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:02.338 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:02.340 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:09:53.789290+0000", 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:02.341 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (1/15)... 2024-09-13T14:10:02.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:02 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:02.492+0000 7ff611786ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:10:02.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:02 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:02.559+0000 7ff611786ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:10:03.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.303+0000 7ff611786ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:10:03.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.418+0000 7ff611786ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:10:03.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.543+0000 7ff611786ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:03.606 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.604+0000 7ff611786ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:10:03.867 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.734+0000 7ff611786ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:10:03.867 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.805+0000 7ff611786ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:10:03.868 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:03.866+0000 7ff611786ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:10:04.144 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.009+0000 7ff611786ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:10:04.144 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.075+0000 7ff611786ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:10:04.144 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.142+0000 7ff611786ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:10:04.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.226+0000 7ff611786ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:04.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.285+0000 7ff611786ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:10:04.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:04.345+0000 7ff611786ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:10:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:04 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1617515308' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:05.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:05.147+0000 7ff611786ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "ad455616-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:05.457 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:05.458 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:05.460 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:05.461 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:05.462 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.462 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:09:53.789290+0000", 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:05.463 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (2/15)... 2024-09-13T14:10:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: Activating manager daemon y 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: mgrmap e2: y(active, starting, since 0.00300868s) 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: Manager daemon y is now available 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' 2024-09-13T14:10:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:05 smithi026 ceph-mon[27770]: from='mgr.14100 172.21.15.26:0/927505882' entity='mgr.y' 2024-09-13T14:10:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:07 smithi026 ceph-mon[27770]: mgrmap e3: y(active, since 1.00671s) 2024-09-13T14:10:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:08 smithi026 ceph-mon[27770]: mgrmap e4: y(active, since 2s) 2024-09-13T14:10:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:08 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3132646725' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "ad455616-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:08.592 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:08.596 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:08.597 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:08.598 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:09:53.789290+0000", 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:08.599 INFO:teuthology.orchestra.run.smithi026.stdout:mgr is available 2024-09-13T14:10:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:09 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2681477872' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:10:13.045 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:13.046 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:10:13.047 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:10:13.047 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:10:13.047 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling cephadm module... 2024-09-13T14:10:16.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/733683286' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T14:10:16.718 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setuser ceph since I am not root 2024-09-13T14:10:16.718 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setgroup ceph since I am not root 2024-09-13T14:10:16.718 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:16.592+0000 7f2f93e0fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:10:16.718 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:16.649+0000 7f2f93e0fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:16.718 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:16.716+0000 7f2f93e0fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:10:16.989 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:16.846+0000 7f2f93e0fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:16.989 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:16.989+0000 7f2f93e0fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:10:17.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:17 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/733683286' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T14:10:17.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:17 smithi026 ceph-mon[27770]: mgrmap e5: y(active, since 11s) 2024-09-13T14:10:17.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:17.259+0000 7f2f93e0fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:10:17.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:17.322+0000 7f2f93e0fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:10:17.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:17.492+0000 7f2f93e0fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:10:18.507 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:18.387+0000 7f2f93e0fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:10:18.508 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:18.455+0000 7f2f93e0fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:10:19.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/317440788' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:10:19.471 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.220+0000 7f2f93e0fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:10:19.471 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.343+0000 7f2f93e0fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:10:19.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.470+0000 7f2f93e0fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:19.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.533+0000 7f2f93e0fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:10:19.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.665+0000 7f2f93e0fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:10:19.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.736+0000 7f2f93e0fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:10:19.792 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:19.793 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2024-09-13T14:10:19.793 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 5... 2024-09-13T14:10:19.999 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.797+0000 7f2f93e0fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:10:19.999 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:19.943+0000 7f2f93e0fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:10:20.293 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:20 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:20.014+0000 7f2f93e0fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:10:20.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:20 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:20.085+0000 7f2f93e0fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:10:20.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:20 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:20.171+0000 7f2f93e0fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:20.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:20 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:20.234+0000 7f2f93e0fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:10:20.294 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:20 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:20.293+0000 7f2f93e0fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:10:21.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:21.074+0000 7f2f93e0fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:10:21.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: Active manager daemon y restarted 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: Activating manager daemon y 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: mgrmap e6: y(active, starting, since 0.00318808s) 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: Manager daemon y is now available 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:21.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:21.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:21.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:10:21.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:21 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:10:22.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:22 smithi026 ceph-mon[27770]: Found migration_current of "None". Setting to last migration. 2024-09-13T14:10:22.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:22 smithi026 ceph-mon[27770]: mgrmap e7: y(active, since 1.005s) 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 5 is available 2024-09-13T14:10:22.722 INFO:teuthology.orchestra.run.smithi026.stdout:Setting orchestrator backend to cephadm... 2024-09-13T14:10:23.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-mon[27770]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:10:23.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-mon[27770]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:23.058+0000 7f2f461fe640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:10:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Traceback (most recent call last): 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._connections.run(self.expiration_interval) 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._run(expiration_interval) 2024-09-13T14:10:23.379 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s = self.context.wrap_socket( 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: return self.sslsocket_class._create( 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self.do_handshake() 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._sslobj.do_handshake() 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:23.380 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:24.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:22] ENGINE Bus STARTING 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:23] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:23] ENGINE Bus STARTED 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:24.658 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:24.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:24 smithi026 ceph-mon[27770]: mgrmap e8: y(active, since 2s) 2024-09-13T14:10:25.310 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T14:10:25.310 INFO:teuthology.orchestra.run.smithi026.stdout:Generating ssh key... 2024-09-13T14:10:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:25 smithi026 ceph-mon[27770]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Generating public/private rsa key pair. 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Your identification has been saved in /tmp/tmpzu6s61aw/key 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Your public key has been saved in /tmp/tmpzu6s61aw/key.pub 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: The key fingerprint is: 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: SHA256:5fPEizn4EmKHH360HOVk9EmvTGQ72loe4yN/6GEpBuM ceph-ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: The key's randomart image is: 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: +---[RSA 3072]----+ 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | | 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | . + | 2024-09-13T14:10:26.363 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | .. = + | 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | o .+ * .| 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | .S ==o= o | 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | + +ooOo.B. | 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | . =.=Eo==++ | 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | +.+ooo=..| 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: | o. +oo | 2024-09-13T14:10:26.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: +----[SHA256]-----+ 2024-09-13T14:10:27.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:27 smithi026 ceph-mon[27770]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:27.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:27 smithi026 ceph-mon[27770]: Generating ssh key... 2024-09-13T14:10:27.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:27 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:27.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:27 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:27.880 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUeX1MjU1KQtvxY4q+mtndjYbaoPdiVOFeIZVMN+7eL0anI0DZ5iI0DD3++WEZPxh4MfHg1/8AqDmbauRkeQEgYmzn1BQMADUNa7UOOZON0AG+W2/tZZO9Ia2s/di8so6TmCV3Ofm91pMkvu0gj2GYejjq8E7X6/tqQ3o/rpj3bGWjXtEBv+LGU/Ays3wnkMFlHahknBOmt6atkoVgJLkxAgnAvg5dQhGbh8JXunJ/W57MHrbzMIbIa4K0h1SKRDXy7lC+R7QZBMhDilFchAFQ8DxeDKEgRxLdvvokPsMzYEzYPYRDTQAU9p9zcjpXi7ChBz6bgN2HWMqBtWeiOCSx5ARsP9yB/lIrtESQkqfbWvNtxczCKz8Es4uRpb9YPm9Rdb9JSP+1y0hD7tcccvTYq8uPncs5x6QxVinBDlqjyQQefGP4vNDQcly7fwfFF8FOEn30escBYVYdiFVxUXvVHu/Okk3Mev41Ao35GB4pF+DNoWCe4pwCEBLkMpLq7s= ceph-ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:27.880 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:10:27.881 INFO:teuthology.orchestra.run.smithi026.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T14:10:27.889 INFO:teuthology.orchestra.run.smithi026.stdout:Adding host smithi026... 2024-09-13T14:10:28.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:28 smithi026 ceph-mon[27770]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:29 smithi026 ceph-mon[27770]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi026", "addr": "172.21.15.26", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:30.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:30 smithi026 ceph-mon[27770]: Deploying cephadm binary to smithi026 2024-09-13T14:10:32.453 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Added host 'smithi026' with addr '172.21.15.26' 2024-09-13T14:10:32.453 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mon service... 2024-09-13T14:10:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:32 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:32 smithi026 ceph-mon[27770]: Added host smithi026 2024-09-13T14:10:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:32 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:34 smithi026 ceph-mon[27770]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:34 smithi026 ceph-mon[27770]: Saving service mon spec with placement count:5 2024-09-13T14:10:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:34 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:34 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:34.842 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T14:10:34.842 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mgr service... 2024-09-13T14:10:36.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:36 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:36.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:36 smithi026 ceph-mon[27770]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:36.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:36 smithi026 ceph-mon[27770]: Saving service mgr spec with placement count:2 2024-09-13T14:10:36.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:36 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:36.257 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T14:10:38.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:38 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2522745842' entity='client.admin' 2024-09-13T14:10:38.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:38 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:39.164 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling the dashboard module... 2024-09-13T14:10:39.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:39 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2684798042' entity='client.admin' 2024-09-13T14:10:39.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:39 smithi026 ceph-mon[27770]: from='mgr.14118 172.21.15.26:0/1112709649' entity='mgr.y' 2024-09-13T14:10:40.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3004277684' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T14:10:40.846 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setuser ceph since I am not root 2024-09-13T14:10:40.846 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setgroup ceph since I am not root 2024-09-13T14:10:40.846 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:40.734+0000 7f09f6b5cec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:10:40.846 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:40.792+0000 7f09f6b5cec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:41.136 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:40.862+0000 7f09f6b5cec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:10:41.136 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:40.991+0000 7f09f6b5cec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:41.137 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:41.134+0000 7f09f6b5cec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:10:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3004277684' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T14:10:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-mon[27770]: mgrmap e9: y(active, since 19s) 2024-09-13T14:10:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:41.407+0000 7f09f6b5cec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:10:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:41.473+0000 7f09f6b5cec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:10:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:41.643+0000 7f09f6b5cec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:10:42.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:42 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/713796741' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:10:42.755 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:42.540+0000 7f09f6b5cec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:10:42.755 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:42.608+0000 7f09f6b5cec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2024-09-13T14:10:42.758 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 9... 2024-09-13T14:10:43.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.362+0000 7f09f6b5cec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:10:43.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.478+0000 7f09f6b5cec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:10:43.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.603+0000 7f09f6b5cec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:43.926 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.665+0000 7f09f6b5cec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:10:43.926 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.795+0000 7f09f6b5cec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:10:43.926 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.864+0000 7f09f6b5cec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:10:43.926 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:43.924+0000 7f09f6b5cec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:10:44.202 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.064+0000 7f09f6b5cec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:10:44.202 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.130+0000 7f09f6b5cec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:10:44.202 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.200+0000 7f09f6b5cec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:10:44.610 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.286+0000 7f09f6b5cec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:44.610 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.347+0000 7f09f6b5cec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:10:44.611 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:44 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:44.408+0000 7f09f6b5cec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:10:45.455 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:45.201+0000 7f09f6b5cec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: Active manager daemon y restarted 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: Activating manager daemon y 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: mgrmap e10: y(active, starting, since 0.00349735s) 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: Manager daemon y is now available 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:45.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:10:45.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 9 is available 2024-09-13T14:10:46.791 INFO:teuthology.orchestra.run.smithi026.stdout:Generating a dashboard self-signed certificate... 2024-09-13T14:10:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-mon[27770]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:10:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-mon[27770]: mgrmap e11: y(active, since 1.00623s) 2024-09-13T14:10:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-mon[27770]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:10:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:47.144+0000 7f09a76ba640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:10:47] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Traceback (most recent call last): 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._connections.run(self.expiration_interval) 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._run(expiration_interval) 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:47.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s = self.context.wrap_socket( 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: return self.sslsocket_class._create( 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self.do_handshake() 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._sslobj.do_handshake() 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:47.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:10:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:10:48.240 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T14:10:48.240 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial admin user... 2024-09-13T14:10:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:47] ENGINE Bus STARTING 2024-09-13T14:10:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:47] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:10:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:47] ENGINE Bus STARTED 2024-09-13T14:10:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: [13/Sep/2024:14:10:47] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:10:48.916 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:48.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:48 smithi026 ceph-mon[27770]: mgrmap e12: y(active, since 2s) 2024-09-13T14:10:49.772 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$sON0PUJV8UWzdEOEC.yRAua8BF.R1g9umUBnwS/oA/6HbyCX1W6NS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726236649, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T14:10:49.772 INFO:teuthology.orchestra.run.smithi026.stdout:Fetching dashboard port number... 2024-09-13T14:10:50.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:50.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:49 smithi026 ceph-mon[27770]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:50.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:50 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/224782813' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T14:10:51.349 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T14:10:51.360 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:10:51.361 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2024-09-13T14:10:51.370 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:10:51.370 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2024-09-13T14:10:51.370 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2024-09-13T14:10:51.370 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph Dashboard is now available at: 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout: URL: https://smithi026.front.sepia.ceph.com:8443/ 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout: User: admin 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout: Password: ao3d7bf2b4 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:51.372 INFO:teuthology.orchestra.run.smithi026.stdout:Saving cluster configuration to /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config directory 2024-09-13T14:10:51.373 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling autotune for osd_memory_target 2024-09-13T14:10:53.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:52 smithi026 ceph-mon[27770]: mgrmap e13: y(active, since 6s) 2024-09-13T14:10:54.173 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.174 INFO:teuthology.orchestra.run.smithi026.stdout: ceph telemetry on 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout:For more information see: 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:10:54.175 INFO:teuthology.orchestra.run.smithi026.stdout:Bootstrap complete. 2024-09-13T14:10:54.217 INFO:tasks.cephadm:Fetching config... 2024-09-13T14:10:54.217 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:10:54.217 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T14:10:54.236 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T14:10:54.236 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:10:54.236 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T14:10:54.293 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T14:10:54.293 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:10:54.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T14:10:54.360 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T14:10:54.360 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:10:54.360 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T14:10:54.416 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T14:10:54.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUeX1MjU1KQtvxY4q+mtndjYbaoPdiVOFeIZVMN+7eL0anI0DZ5iI0DD3++WEZPxh4MfHg1/8AqDmbauRkeQEgYmzn1BQMADUNa7UOOZON0AG+W2/tZZO9Ia2s/di8so6TmCV3Ofm91pMkvu0gj2GYejjq8E7X6/tqQ3o/rpj3bGWjXtEBv+LGU/Ays3wnkMFlHahknBOmt6atkoVgJLkxAgnAvg5dQhGbh8JXunJ/W57MHrbzMIbIa4K0h1SKRDXy7lC+R7QZBMhDilFchAFQ8DxeDKEgRxLdvvokPsMzYEzYPYRDTQAU9p9zcjpXi7ChBz6bgN2HWMqBtWeiOCSx5ARsP9yB/lIrtESQkqfbWvNtxczCKz8Es4uRpb9YPm9Rdb9JSP+1y0hD7tcccvTYq8uPncs5x6QxVinBDlqjyQQefGP4vNDQcly7fwfFF8FOEn30escBYVYdiFVxUXvVHu/Okk3Mev41Ao35GB4pF+DNoWCe4pwCEBLkMpLq7s= ceph-ad455616-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:10:54.493 INFO:teuthology.orchestra.run.smithi026.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUeX1MjU1KQtvxY4q+mtndjYbaoPdiVOFeIZVMN+7eL0anI0DZ5iI0DD3++WEZPxh4MfHg1/8AqDmbauRkeQEgYmzn1BQMADUNa7UOOZON0AG+W2/tZZO9Ia2s/di8so6TmCV3Ofm91pMkvu0gj2GYejjq8E7X6/tqQ3o/rpj3bGWjXtEBv+LGU/Ays3wnkMFlHahknBOmt6atkoVgJLkxAgnAvg5dQhGbh8JXunJ/W57MHrbzMIbIa4K0h1SKRDXy7lC+R7QZBMhDilFchAFQ8DxeDKEgRxLdvvokPsMzYEzYPYRDTQAU9p9zcjpXi7ChBz6bgN2HWMqBtWeiOCSx5ARsP9yB/lIrtESQkqfbWvNtxczCKz8Es4uRpb9YPm9Rdb9JSP+1y0hD7tcccvTYq8uPncs5x6QxVinBDlqjyQQefGP4vNDQcly7fwfFF8FOEn30escBYVYdiFVxUXvVHu/Okk3Mev41Ao35GB4pF+DNoWCe4pwCEBLkMpLq7s= ceph-ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:54.515 DEBUG:teuthology.orchestra.run.smithi145:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUeX1MjU1KQtvxY4q+mtndjYbaoPdiVOFeIZVMN+7eL0anI0DZ5iI0DD3++WEZPxh4MfHg1/8AqDmbauRkeQEgYmzn1BQMADUNa7UOOZON0AG+W2/tZZO9Ia2s/di8so6TmCV3Ofm91pMkvu0gj2GYejjq8E7X6/tqQ3o/rpj3bGWjXtEBv+LGU/Ays3wnkMFlHahknBOmt6atkoVgJLkxAgnAvg5dQhGbh8JXunJ/W57MHrbzMIbIa4K0h1SKRDXy7lC+R7QZBMhDilFchAFQ8DxeDKEgRxLdvvokPsMzYEzYPYRDTQAU9p9zcjpXi7ChBz6bgN2HWMqBtWeiOCSx5ARsP9yB/lIrtESQkqfbWvNtxczCKz8Es4uRpb9YPm9Rdb9JSP+1y0hD7tcccvTYq8uPncs5x6QxVinBDlqjyQQefGP4vNDQcly7fwfFF8FOEn30escBYVYdiFVxUXvVHu/Okk3Mev41Ao35GB4pF+DNoWCe4pwCEBLkMpLq7s= ceph-ad455616-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:10:54.804 INFO:teuthology.orchestra.run.smithi145.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUeX1MjU1KQtvxY4q+mtndjYbaoPdiVOFeIZVMN+7eL0anI0DZ5iI0DD3++WEZPxh4MfHg1/8AqDmbauRkeQEgYmzn1BQMADUNa7UOOZON0AG+W2/tZZO9Ia2s/di8so6TmCV3Ofm91pMkvu0gj2GYejjq8E7X6/tqQ3o/rpj3bGWjXtEBv+LGU/Ays3wnkMFlHahknBOmt6atkoVgJLkxAgnAvg5dQhGbh8JXunJ/W57MHrbzMIbIa4K0h1SKRDXy7lC+R7QZBMhDilFchAFQ8DxeDKEgRxLdvvokPsMzYEzYPYRDTQAU9p9zcjpXi7ChBz6bgN2HWMqBtWeiOCSx5ARsP9yB/lIrtESQkqfbWvNtxczCKz8Es4uRpb9YPm9Rdb9JSP+1y0hD7tcccvTYq8uPncs5x6QxVinBDlqjyQQefGP4vNDQcly7fwfFF8FOEn30escBYVYdiFVxUXvVHu/Okk3Mev41Ao35GB4pF+DNoWCe4pwCEBLkMpLq7s= ceph-ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:54.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T14:10:54.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:54 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4169376200' entity='client.admin' 2024-09-13T14:10:55.081 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi026", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:57.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:10:58.620 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T14:10:58.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T14:10:59.027 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:10:59.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:59 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2328825462' entity='client.admin' 2024-09-13T14:10:59.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:59 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:59.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:10:59 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:01.339 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi145 2024-09-13T14:11:01.339 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:11:01.339 DEBUG:teuthology.orchestra.run.smithi145:> dd of=/etc/ceph/ceph.conf 2024-09-13T14:11:01.359 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:11:01.359 DEBUG:teuthology.orchestra.run.smithi145:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:01.419 INFO:tasks.cephadm:Adding host smithi145 to orchestrator... 2024-09-13T14:11:01.419 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch host add smithi145 2024-09-13T14:11:01.693 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:11:01.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:01.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:01.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:11:01.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:01.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:01.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:04.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:03 smithi026 ceph-mon[27770]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi145", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:05.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:04 smithi026 ceph-mon[27770]: Deploying cephadm binary to smithi145 2024-09-13T14:11:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:06 smithi026 ceph-mon[27770]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:06.690 INFO:teuthology.orchestra.run.smithi026.stdout:Added host 'smithi145' with addr '172.21.15.145' 2024-09-13T14:11:07.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T14:11:07.570 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:07.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:07.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: Added host smithi145 2024-09-13T14:11:07.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:07.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:07.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:07.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:09.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:09 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:09.250 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:11:09.250 INFO:teuthology.orchestra.run.smithi026.stdout:[{"addr": "172.21.15.26", "hostname": "smithi026", "labels": [], "status": ""}, {"addr": "172.21.15.145", "hostname": "smithi145", "labels": [], "status": ""}] 2024-09-13T14:11:09.846 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T14:11:09.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T14:11:10.095 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:10.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:10 smithi026 ceph-mon[27770]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:10.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:10 smithi026 ceph-mon[27770]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:11:11.952 INFO:teuthology.orchestra.run.smithi026.stderr:adjusted tunables profile to default 2024-09-13T14:11:12.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi145", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/982932091' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:12.573 INFO:tasks.cephadm:Adding mon.a on smithi026 2024-09-13T14:11:12.573 INFO:tasks.cephadm:Adding mon.c on smithi026 2024-09-13T14:11:12.574 INFO:tasks.cephadm:Adding mon.b on smithi145 2024-09-13T14:11:12.574 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b' 2024-09-13T14:11:12.808 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:12 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/982932091' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:11:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:12 smithi026 ceph-mon[27770]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:14.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:13 smithi026 ceph-mon[27770]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:14.439 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled mon update... 2024-09-13T14:11:14.952 DEBUG:teuthology.orchestra.run.smithi026:mon.c> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T14:11:14.955 DEBUG:teuthology.orchestra.run.smithi145:mon.b> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T14:11:14.957 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:11:14.957 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:11:15.769 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:15.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:15.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b;count:3 2024-09-13T14:11:15.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:15.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:15.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:15 smithi026 ceph-mon[27770]: Deploying daemon mon.b on smithi145 2024-09-13T14:11:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:16 smithi026 ceph-mon[27770]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:18.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:18 smithi026 ceph-mon[27770]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:19.017 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:11:19.017 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":1,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:09:50.810274Z","created":"2024-09-13T14:09:50.810274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T14:11:19.017 INFO:teuthology.orchestra.run.smithi145.stderr:dumped monmap epoch 1 2024-09-13T14:11:19.246 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:19 smithi145 systemd[1]: Starting Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:11:19.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:19 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/2276879933' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:20.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:19 smithi145 podman[31019]: 2024-09-13 14:11:19.818144003 +0000 UTC m=+0.015612935 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:11:20.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:19 smithi145 podman[31019]: 2024-09-13 14:11:19.922074437 +0000 UTC m=+0.119543365 container create 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:11:20.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 podman[31019]: 2024-09-13 14:11:20.116273701 +0000 UTC m=+0.313742636 container init 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:11:20.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 podman[31019]: 2024-09-13 14:11:20.120831339 +0000 UTC m=+0.318300267 container start 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: pidfile_write: ignore empty --pid-file 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: load: jerasure load: lrc 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: DB SUMMARY 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: DB Session ID: 95FFZ6ABI9560I7U6GE3 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:11:20.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:11:20.169 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T14:11:20.169 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-13T14:11:20.169 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:11:20.170 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:11:20.170 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.env: 0x5622fb60a440 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.fs: Posix File System 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.info_log: 0x5622fbf04ec0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.statistics: (nil) 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.use_fsync: 0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:11:20.171 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.db_log_dir: 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:11:20.172 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.write_buffer_manager: 0x5622fbfc27e0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:11:20.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.unordered_write: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.row_cache: None 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.wal_filter: None 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:11:20.174 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:11:20.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_open_files: -1 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:11:20.176 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Compression algorithms supported: 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kZSTD supported: 0 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T14:11:20.177 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.merge_operator: 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_filter: None 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5622fbeaba38) 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:11:20.178 INFO:journalctl@ceph.mon.b.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_type: 0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: data_block_index_type: 0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_shortening: 1 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: hash_index_allow_collision: 1 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: checksum: 1 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: no_block_cache: 0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache: 0x5622fbee11f0 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_options: 2024-09-13T14:11:20.179 INFO:journalctl@ceph.mon.b.smithi145.stdout: capacity : 536870912 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: num_shard_bits : 4 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: strict_capacity_limit : 0 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_compressed: (nil) 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: persistent_cache: (nil) 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_size: 4096 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_size_deviation: 10 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_restart_interval: 16 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_block_restart_interval: 1 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: metadata_block_size: 4096 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: partition_filters: 0 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: use_delta_encoding: 1 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi145.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: whole_key_filtering: 1 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: verify_compression: 0 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: format_version: 4 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: enable_index_compression: 1 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_align: 0 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression: NoCompression 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.num_levels: 7 2024-09-13T14:11:20.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:11:20.182 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:11:20.183 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:11:20.184 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:11:20.185 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.ttl: 2592000 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:11:20.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236680144876, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236680145181, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236680, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "43575f1d-eea2-4254-9118-030aecac2e0f", "db_session_id": "95FFZ6ABI9560I7U6GE3"}} 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236680145534, "job": 1, "event": "recovery_finished"} 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5622fbf14700 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: DB pointer 0x5622fbee4000 2024-09-13T14:11:20.187 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** DB Stats ** 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:20.188 INFO:journalctl@ceph.mon.b.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:20.189 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.190 INFO:journalctl@ceph.mon.b.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:20.191 INFO:journalctl@ceph.mon.b.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: using public_addr v2:172.21.15.145:0/0 -> [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: starting mon.b rank -1 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(???) e0 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).mds e1 new map 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: e1 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: legacy client fscid: -1 2024-09-13T14:11:20.192 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout: No filesystems configured 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:20.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b;count:3 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: Deploying daemon mon.b on smithi145 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/2276879933' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:20.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 ceph-mon[31057]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:11:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 bash[31019]: 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db 2024-09-13T14:11:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:20 smithi145 systemd[1]: Started Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:11:20.692 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:11:20.692 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:11:20.951 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:25.603 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:11:25.604 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":2,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:11:20.171291Z","created":"2024-09-13T14:09:50.810274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T14:11:25.604 INFO:teuthology.orchestra.run.smithi145.stderr:dumped monmap epoch 2 2024-09-13T14:11:25.604 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.604 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:25.604 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: mon.a calling monitor election 2024-09-13T14:11:25.604 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: mon.b calling monitor election 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: fsmap 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: mgrmap e13: y(active, since 39s) 2024-09-13T14:11:25.605 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: overall HEALTH_OK 2024-09-13T14:11:25.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:25.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: mon.a calling monitor election 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: mon.b calling monitor election 2024-09-13T14:11:25.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: fsmap 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: mgrmap e13: y(active, since 39s) 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: overall HEALTH_OK 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:25.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:26 smithi145 ceph-mon[31057]: Deploying daemon mon.c on smithi026 2024-09-13T14:11:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:26 smithi145 ceph-mon[31057]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:26 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/2100437044' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:26.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:26 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:26.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:26 smithi026 ceph-mon[27770]: Deploying daemon mon.c on smithi026 2024-09-13T14:11:26.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:26 smithi026 ceph-mon[27770]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:26.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:26 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/2100437044' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:26.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:26 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:27.196 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:11:27.197 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:11:27.444 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:28.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 podman[39144]: 2024-09-13 14:11:28.075862124 +0000 UTC m=+0.117150213 container create 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-13T14:11:28.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:28 smithi145 ceph-mon[31057]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[27770]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 podman[39144]: 2024-09-13 14:11:28.278125617 +0000 UTC m=+0.319413674 container init 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 podman[39144]: 2024-09-13 14:11:28.283121769 +0000 UTC m=+0.324409817 container start 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pidfile_write: ignore empty --pid-file 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: load: jerasure load: lrc 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:11:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: DB SUMMARY 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: DB Session ID: 0AEI0DD3DFIMJGLHY6LC 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-13T14:11:28.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.env: 0x55d43477c440 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.fs: Posix File System 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.info_log: 0x55d4369a6ec0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.statistics: (nil) 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.use_fsync: 0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:11:28.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.db_log_dir: 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:11:28.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.write_buffer_manager: 0x55d436a647b0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:11:28.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.unordered_write: 0 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.row_cache: None 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.wal_filter: None 2024-09-13T14:11:28.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:11:28.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:11:28.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_open_files: -1 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Compression algorithms supported: 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kZSTD supported: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:11:28.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.merge_operator: 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_filter: None 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:11:28.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d43694da38) 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:11:28.677 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: hash_index_allow_collision: 1 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 1 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x55d4369831f0 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2024-09-13T14:11:28.678 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:11:28.679 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 4 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression: NoCompression 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:11:28.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.num_levels: 7 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:11:28.681 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:11:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:11:28.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:11:28.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:11:28.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.ttl: 2592000 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:11:28.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236688306981, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236688307282, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236688, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a0c2a6c1-eb6c-4e6d-ab83-f52a008f406e", "db_session_id": "0AEI0DD3DFIMJGLHY6LC"}} 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236688307623, "job": 1, "event": "recovery_finished"} 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d4369b6700 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: DB pointer 0x55d436986000 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2024-09-13T14:11:28.687 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:28.688 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:28.689 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.690 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-09-13T14:11:28.691 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.692 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: using public_addrv [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: starting mon.c rank -1 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(???) e0 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: e1 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: -1 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout: No filesystems configured 2024-09-13T14:11:28.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:28.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi145:172.21.15.145=b;count:3 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: Deploying daemon mon.b on smithi145 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/2276879933' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.a calling monitor election 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.b calling monitor election 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: fsmap 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mgrmap e13: y(active, since 39s) 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: overall HEALTH_OK 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: Deploying daemon mon.c on smithi026 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/2100437044' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 ceph-mon[39182]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:11:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 bash[39144]: 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 2024-09-13T14:11:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:28 smithi026 systemd[1]: Started Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:11:33.612 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:11:33.612 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":3,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:11:28.329710Z","created":"2024-09-13T14:09:50.810274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-13T14:11:33.613 INFO:teuthology.orchestra.run.smithi145.stderr:dumped monmap epoch 3 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: mon.a calling monitor election 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: mon.b calling monitor election 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: mon.c calling monitor election 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.651 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: fsmap 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: mgrmap e13: y(active, since 48s) 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: overall HEALTH_OK 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: mon.a calling monitor election 2024-09-13T14:11:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: mon.b calling monitor election 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: mon.c calling monitor election 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:11:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: fsmap 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: mgrmap e13: y(active, since 48s) 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: overall HEALTH_OK 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: mon.a calling monitor election 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: mon.b calling monitor election 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: mon.c calling monitor election 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: fsmap 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: mgrmap e13: y(active, since 48s) 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: overall HEALTH_OK 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:34.215 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T14:11:34.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T14:11:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:34 smithi145 ceph-mon[31057]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:34 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:34 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/1520694886' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:34 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:34.470 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:34.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[39182]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:34.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/1520694886' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[27770]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/1520694886' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:11:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:34 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:11:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:35.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:35 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:35 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.325 INFO:teuthology.orchestra.run.smithi026.stdout:# minimal ceph.conf for ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:36.326 INFO:teuthology.orchestra.run.smithi026.stdout:[global] 2024-09-13T14:11:36.326 INFO:teuthology.orchestra.run.smithi026.stdout: fsid = ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:36.326 INFO:teuthology.orchestra.run.smithi026.stdout: mon_host = [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2024-09-13T14:11:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:11:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:11:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:11:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:36 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2604774740' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:11:36.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:11:36.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:36.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:11:36.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:36.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:36.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:11:36.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2604774740' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:11:36.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:11:36.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:36.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:36.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:36.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:11:36.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:36.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:36.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:11:36.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:36 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2604774740' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.924 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T14:11:36.925 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:11:36.925 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:11:36.955 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:11:36.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:37.024 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:11:37.024 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:11:37.056 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:11:37.056 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:37.123 INFO:tasks.cephadm:Adding mgr.y on smithi026 2024-09-13T14:11:37.123 INFO:tasks.cephadm:Adding mgr.x on smithi145 2024-09-13T14:11:37.124 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi026=y;smithi145=x' 2024-09-13T14:11:37.398 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:38.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:38.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:38.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:39.058 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled mgr update... 2024-09-13T14:11:39.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:11:39.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:11:39.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.649 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.649 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.649 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:39.649 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:39.649 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:39.673 DEBUG:teuthology.orchestra.run.smithi145:mgr.x> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T14:11:39.675 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T14:11:39.676 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T14:11:39.676 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:11:39.676 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:11:39.696 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:11:39.696 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_1 2024-09-13T14:11:39.755 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:11:39.755 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 953 Links: 1 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-09-13 14:11:27.796066335 +0000 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:39.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:39.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:11:39.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:39.829 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-09-13T14:11:39.829 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-09-13T14:11:39.829 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000123484 s, 4.1 MB/s 2024-09-13T14:11:39.831 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:11:39.890 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_2 2024-09-13T14:11:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-09-13 14:11:27.796066335 +0000 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-09-13 14:07:12.592846075 +0000 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-09-13 14:07:12.592846075 +0000 2024-09-13T14:11:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-09-13 14:07:12.592846075 +0000 2024-09-13T14:11:39.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:11:40.019 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-09-13T14:11:40.019 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-09-13T14:11:40.020 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000135062 s, 3.8 MB/s 2024-09-13T14:11:40.021 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:11:40.081 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_3 2024-09-13T14:11:40.140 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:11:40.140 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:40.140 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 956 Links: 1 2024-09-13T14:11:40.140 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:40.141 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:40.141 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-09-13 14:11:27.797066320 +0000 2024-09-13T14:11:40.141 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:40.141 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:40.141 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-09-13 14:07:12.589846119 +0000 2024-09-13T14:11:40.142 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:11:40.208 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-09-13T14:11:40.208 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-09-13T14:11:40.208 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.00012219 s, 4.2 MB/s 2024-09-13T14:11:40.210 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:11:40.268 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_4 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-09-13 14:11:27.797066320 +0000 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-09-13 14:07:12.591846090 +0000 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-09-13 14:07:12.591846090 +0000 2024-09-13T14:11:40.326 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-09-13 14:07:12.591846090 +0000 2024-09-13T14:11:40.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:11:40.393 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-09-13T14:11:40.393 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-09-13T14:11:40.393 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000128608 s, 4.0 MB/s 2024-09-13T14:11:40.394 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:11:40.453 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:11:40.453 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:11:40.470 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:11:40.471 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_1 2024-09-13T14:11:40.526 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-09-13 14:11:19.086714936 +0000 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.527 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.527 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:11:40.591 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-09-13T14:11:40.591 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-09-13T14:11:40.591 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000195174 s, 2.6 MB/s 2024-09-13T14:11:40.592 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:11:40.652 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_2 2024-09-13T14:11:40.705 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:11:40.705 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:40.705 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 977 Links: 1 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-09-13 14:11:19.086714936 +0000 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.706 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-09-13 14:07:13.102295616 +0000 2024-09-13T14:11:40.707 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:11:40.738 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:40 smithi145 ceph-mon[31057]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi145=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:40.738 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:40 smithi145 ceph-mon[31057]: Saving service mgr spec with placement smithi026=y;smithi145=x;count:2 2024-09-13T14:11:40.738 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:40 smithi145 ceph-mon[31057]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:40.739 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:40 smithi145 ceph-mon[31057]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:11:40.739 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:40 smithi145 ceph-mon[31057]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:11:40.762 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-09-13T14:11:40.762 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-09-13T14:11:40.762 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000203103 s, 2.5 MB/s 2024-09-13T14:11:40.764 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:11:40.820 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_3 2024-09-13T14:11:40.875 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-09-13 14:11:19.086714936 +0000 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-09-13 14:07:13.103295593 +0000 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-09-13 14:07:13.103295593 +0000 2024-09-13T14:11:40.876 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-09-13 14:07:13.103295593 +0000 2024-09-13T14:11:40.876 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:11:40.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[27770]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi145=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[27770]: Saving service mgr spec with placement smithi026=y;smithi145=x;count:2 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[27770]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[27770]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[27770]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[39182]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi145=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[39182]: Saving service mgr spec with placement smithi026=y;smithi145=x;count:2 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[39182]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[39182]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:11:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:40 smithi026 ceph-mon[39182]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:11:40.940 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-09-13T14:11:40.940 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-09-13T14:11:40.940 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000281582 s, 1.8 MB/s 2024-09-13T14:11:40.941 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:11:40.998 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_4 2024-09-13T14:11:41.053 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:11:41.053 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:11:41.053 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-13T14:11:41.053 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:11:41.053 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:11:41.054 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-09-13 14:11:19.087714914 +0000 2024-09-13T14:11:41.054 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-09-13 14:07:13.101295639 +0000 2024-09-13T14:11:41.054 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-09-13 14:07:13.101295639 +0000 2024-09-13T14:11:41.054 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-09-13 14:07:13.101295639 +0000 2024-09-13T14:11:41.054 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:11:41.119 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-09-13T14:11:41.120 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-09-13T14:11:41.120 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000328066 s, 1.6 MB/s 2024-09-13T14:11:41.121 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:11:41.178 INFO:tasks.cephadm:Deploying osd.0 on smithi026 with /dev/vg_nvme/lv_4... 2024-09-13T14:11:41.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:11:41.437 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:41.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:41.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:41.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: Deploying daemon mgr.x on smithi145 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[27770]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:41.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: Deploying daemon mgr.x on smithi145 2024-09-13T14:11:41.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:41 smithi026 ceph-mon[39182]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:11:42.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:42.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:42.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: Deploying daemon mgr.x on smithi145 2024-09-13T14:11:42.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:41 smithi145 ceph-mon[31057]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:42.803 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 podman[33093]: 2024-09-13 14:11:42.615071637 +0000 UTC m=+0.015398730 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:11:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 podman[33093]: 2024-09-13 14:11:42.8982925 +0000 UTC m=+0.298619584 container init 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T14:11:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 podman[33093]: 2024-09-13 14:11:42.902982815 +0000 UTC m=+0.303309900 container start 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T14:11:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 ceph-mgr[33131]: -- 172.21.15.145:0/3101121272 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5571e86d4340 con 0x5571e86e1000 2024-09-13T14:11:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 bash[33093]: 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 2024-09-13T14:11:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:42 smithi145 systemd[1]: Started Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:11:43.206 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.017+0000 7fdb7a461ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:11:43.206 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.077+0000 7fdb7a461ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:11:43.206 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.146+0000 7fdb7a461ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:11:43.688 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.275+0000 7fdb7a461ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:11:43.688 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.421+0000 7fdb7a461ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:11:43.688 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.687+0000 7fdb7a461ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:11:43.820 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:11:43.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_4 2024-09-13T14:11:43.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.750+0000 7fdb7a461ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:11:43.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:43.915+0000 7fdb7a461ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:11:44.095 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[27770]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:44 smithi026 ceph-mon[39182]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-mon[31057]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:45.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:44.760+0000 7fdb7a461ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:11:45.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:44.822+0000 7fdb7a461ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:11:45.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[27770]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:45 smithi026 ceph-mon[39182]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:11:45.519 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.519 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.519 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:45.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-mon[31057]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:11:45.802 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.518+0000 7fdb7a461ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:11:45.802 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.627+0000 7fdb7a461ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:11:45.802 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.743+0000 7fdb7a461ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:11:46.183 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.801+0000 7fdb7a461ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:11:46.183 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.929+0000 7fdb7a461ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:11:46.183 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:45.994+0000 7fdb7a461ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:11:46.184 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.051+0000 7fdb7a461ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:11:46.184 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.182+0000 7fdb7a461ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:46.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.247+0000 7fdb7a461ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:11:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.314+0000 7fdb7a461ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:11:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.397+0000 7fdb7a461ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:11:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:46.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:46.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.456+0000 7fdb7a461ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:11:46.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:46.513+0000 7fdb7a461ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:11:47.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: Standby manager daemon x started 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:11:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:47 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:11:47.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:11:47.241+0000 7fdb7a461ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:11:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:11:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:11:47.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:47 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/3346484473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[39182]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[39182]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[39182]: mgrmap e14: y(active, since 62s), standbys: x 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[27770]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[27770]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[27770]: mgrmap e14: y(active, since 62s), standbys: x 2024-09-13T14:11:48.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:11:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:48 smithi145 ceph-mon[31057]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:48 smithi145 ceph-mon[31057]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:48 smithi145 ceph-mon[31057]: mgrmap e14: y(active, since 62s), standbys: x 2024-09-13T14:11:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:48 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:11:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]: dispatch 2024-09-13T14:11:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]': finished 2024-09-13T14:11:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[39182]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:11:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:11:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]: dispatch 2024-09-13T14:11:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]': finished 2024-09-13T14:11:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[27770]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:11:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:11:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:49 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]: dispatch 2024-09-13T14:11:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:49 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3869154059' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff918df3-c46e-4975-9452-aa5b2025532d"}]': finished 2024-09-13T14:11:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:49 smithi145 ceph-mon[31057]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:11:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:49 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:11:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:50 smithi026 ceph-mon[27770]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:50 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/38206480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:50 smithi026 ceph-mon[39182]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:50 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/38206480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:50 smithi145 ceph-mon[31057]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:50 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/38206480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:52 smithi026 ceph-mon[27770]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:52.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:52 smithi026 ceph-mon[39182]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:52 smithi145 ceph-mon[31057]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:54.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:54 smithi026 ceph-mon[27770]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:54.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:54 smithi026 ceph-mon[39182]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:54 smithi145 ceph-mon[31057]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:55.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:55 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:11:55.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:55 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:55.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:55 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:11:55.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:55 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:55 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:11:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:55 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:56 smithi026 ceph-mon[27770]: Deploying daemon osd.0 on smithi026 2024-09-13T14:11:56.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:56 smithi026 ceph-mon[27770]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:56.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:56 smithi026 ceph-mon[39182]: Deploying daemon osd.0 on smithi026 2024-09-13T14:11:56.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:56 smithi026 ceph-mon[39182]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:56 smithi145 ceph-mon[31057]: Deploying daemon osd.0 on smithi026 2024-09-13T14:11:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:56 smithi145 ceph-mon[31057]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:58.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:11:58 smithi026 ceph-mon[27770]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:58.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:11:58 smithi026 ceph-mon[39182]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:11:58 smithi145 ceph-mon[31057]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:00.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[27770]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:00.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:00.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:00.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:00.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[39182]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:00.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:00.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:00.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:00 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:00 smithi145 ceph-mon[31057]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:00 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:00 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:00 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:02 smithi026 ceph-mon[27770]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:02 smithi026 ceph-mon[39182]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:02 smithi145 ceph-mon[31057]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:03.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:03 smithi026 ceph-mon[27770]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:03.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:03 smithi026 ceph-mon[27770]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:03.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:03 smithi026 ceph-mon[39182]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:03.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:03 smithi026 ceph-mon[39182]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:03.384 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 0 on host 'smithi026' 2024-09-13T14:12:03.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:03 smithi145 ceph-mon[31057]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:03.607 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:03 smithi145 ceph-mon[31057]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:12:04.098 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T14:12:04.100 INFO:tasks.cephadm:Deploying osd.1 on smithi026 with /dev/vg_nvme/lv_3... 2024-09-13T14:12:04.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:12:04.330 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[43451]: 2024-09-13T14:12:04.285+0000 7f4107158640 -1 osd.0 0 waiting for initial osdmap 2024-09-13T14:12:04.330 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[43451]: 2024-09-13T14:12:04.287+0000 7f4103f93640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:04.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:05.318 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:05.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[39182]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[39182]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[27770]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[27770]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:05 smithi145 ceph-mon[31057]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:05 smithi145 ceph-mon[31057]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:12:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:12:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262] boot 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262] boot 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:12:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: osd.0 [v2:172.21.15.26:6802/632585262,v1:172.21.15.26:6803/632585262] boot 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:12:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:07 smithi026 ceph-mon[27770]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:12:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:07 smithi026 ceph-mon[39182]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:12:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:07 smithi145 ceph-mon[31057]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:12:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:09.137 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:12:09.159 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_3 2024-09-13T14:12:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:09 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi026 2024-09-13T14:12:09.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:09 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 16068M 2024-09-13T14:12:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:09 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi026 2024-09-13T14:12:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:09 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 16068M 2024-09-13T14:12:09.416 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:09 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi026 2024-09-13T14:12:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:09 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 16068M 2024-09-13T14:12:10.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:10 smithi026 ceph-mon[27770]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:10.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:10 smithi026 ceph-mon[39182]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:10 smithi145 ceph-mon[31057]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:11.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:11.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:11.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:11.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:11.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:11.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:11 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:11 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:11 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:11 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:12.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:12 smithi026 ceph-mon[27770]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:12.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:12 smithi026 ceph-mon[27770]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:12.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:12 smithi026 ceph-mon[39182]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:12.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:12 smithi026 ceph-mon[39182]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:12 smithi145 ceph-mon[31057]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:12 smithi145 ceph-mon[31057]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]: dispatch 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]': finished 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:14.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2333818189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]: dispatch 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]': finished 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:14.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:14 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2333818189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]: dispatch 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/529134100' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "247b896e-5e0a-47ef-b8c6-61618b71477a"}]': finished 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:14 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2333818189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:16 smithi026 ceph-mon[27770]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:16.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:16 smithi026 ceph-mon[39182]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:16 smithi145 ceph-mon[31057]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:18.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:18 smithi026 ceph-mon[39182]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:18.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:18 smithi026 ceph-mon[27770]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:18 smithi145 ceph-mon[31057]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[39182]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[27770]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:12:20.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:20 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:20 smithi145 ceph-mon[31057]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:12:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:21.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:21 smithi026 ceph-mon[27770]: Deploying daemon osd.1 on smithi026 2024-09-13T14:12:21.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:21 smithi026 ceph-mon[39182]: Deploying daemon osd.1 on smithi026 2024-09-13T14:12:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:21 smithi145 ceph-mon[31057]: Deploying daemon osd.1 on smithi026 2024-09-13T14:12:22.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:22 smithi026 ceph-mon[27770]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:22.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:22 smithi026 ceph-mon[39182]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:22 smithi145 ceph-mon[31057]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:24.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:24 smithi026 ceph-mon[27770]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:24.463 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:24 smithi026 ceph-mon[39182]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:24 smithi145 ceph-mon[31057]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:25.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:25.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:25.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:25.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:25.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:25.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:25 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:25.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:25.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:25.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:25 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:26 smithi026 ceph-mon[27770]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:26 smithi026 ceph-mon[39182]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:26 smithi145 ceph-mon[31057]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:28.381 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 1 on host 'smithi026' 2024-09-13T14:12:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:28 smithi026 ceph-mon[27770]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:28 smithi026 ceph-mon[27770]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:12:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:28 smithi026 ceph-mon[39182]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:28 smithi026 ceph-mon[39182]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:12:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:28 smithi145 ceph-mon[31057]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:28 smithi145 ceph-mon[31057]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:12:28.977 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:12:28.980 INFO:tasks.cephadm:Deploying osd.2 on smithi026 with /dev/vg_nvme/lv_2... 2024-09-13T14:12:28.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:12:29.290 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:29.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.616 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[49641]: 2024-09-13T14:12:29.340+0000 7f8e5f340640 -1 osd.1 0 waiting for initial osdmap 2024-09-13T14:12:29.616 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[49641]: 2024-09-13T14:12:29.343+0000 7f8e5a177640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:12:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:12:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:12:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:29.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:30.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:30 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: from='osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:30 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:12:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[27770]: osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120] boot 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[27770]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[39182]: osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120] boot 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[39182]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:12:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:31 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:31.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:31 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:12:31.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:31 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:12:31.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:31 smithi145 ceph-mon[31057]: osd.1 [v2:172.21.15.26:6810/3428658120,v1:172.21.15.26:6811/3428658120] boot 2024-09-13T14:12:31.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:31 smithi145 ceph-mon[31057]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:12:31.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:31 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:32.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:32 smithi026 ceph-mon[39182]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:32.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:32 smithi026 ceph-mon[39182]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:12:32.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:32 smithi026 ceph-mon[27770]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:32.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:32 smithi026 ceph-mon[27770]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:12:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:32 smithi145 ceph-mon[31057]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:12:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:32 smithi145 ceph-mon[31057]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:12:32.861 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:12:32.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_2 2024-09-13T14:12:33.478 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:34.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:34 smithi026 ceph-mon[27770]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:34.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:34 smithi026 ceph-mon[39182]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:34 smithi145 ceph-mon[31057]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:36 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:36.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:36 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:37.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:37 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi026 2024-09-13T14:12:37.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:37 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 8034M 2024-09-13T14:12:37.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:37 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi026 2024-09-13T14:12:37.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:37 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 8034M 2024-09-13T14:12:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:37 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi026 2024-09-13T14:12:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:37 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 8034M 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[27770]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]: dispatch 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]': finished 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[27770]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[39182]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]: dispatch 2024-09-13T14:12:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]': finished 2024-09-13T14:12:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[39182]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:12:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:38 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:38 smithi145 ceph-mon[31057]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:38 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]: dispatch 2024-09-13T14:12:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:38 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3836860733' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbb32835-8bfe-413c-86dc-b733e8800551"}]': finished 2024-09-13T14:12:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:38 smithi145 ceph-mon[31057]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:12:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:38 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:39.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:39 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2953121203' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:39 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2953121203' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:39 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2953121203' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:12:40.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:40 smithi026 ceph-mon[39182]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:40.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:40 smithi026 ceph-mon[27770]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:40 smithi145 ceph-mon[31057]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:42 smithi026 ceph-mon[27770]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:42.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:42 smithi026 ceph-mon[39182]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:42 smithi145 ceph-mon[31057]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:44.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[27770]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:44.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:12:44.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:44.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[39182]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:44.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:12:44.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:44 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:44 smithi145 ceph-mon[31057]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:12:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:44 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:45.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:45 smithi026 ceph-mon[27770]: Deploying daemon osd.2 on smithi026 2024-09-13T14:12:45.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:45 smithi026 ceph-mon[39182]: Deploying daemon osd.2 on smithi026 2024-09-13T14:12:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:45 smithi145 ceph-mon[31057]: Deploying daemon osd.2 on smithi026 2024-09-13T14:12:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:46 smithi026 ceph-mon[27770]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:46.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:46 smithi026 ceph-mon[39182]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:46 smithi145 ceph-mon[31057]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:48.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:48 smithi026 ceph-mon[27770]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:48.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:48 smithi026 ceph-mon[39182]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:48 smithi145 ceph-mon[31057]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:49.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:49.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:49.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:49.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:49.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:49.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:49 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:49 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:49 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:49 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:50 smithi026 ceph-mon[27770]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:50 smithi026 ceph-mon[39182]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:50 smithi145 ceph-mon[31057]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:52.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:52.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[27770]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:52 smithi026 ceph-mon[39182]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:12:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:52.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:52.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:52 smithi145 ceph-mon[31057]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:12:52.723 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 2 on host 'smithi026' 2024-09-13T14:12:53.413 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T14:12:53.415 INFO:tasks.cephadm:Deploying osd.3 on smithi026 with /dev/vg_nvme/lv_1... 2024-09-13T14:12:53.415 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:12:53.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:53.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[27770]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:12:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:12:53.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:53.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:53 smithi026 ceph-mon[39182]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:53 smithi145 ceph-mon[31057]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:12:54.171 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:54.298 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[55591]: 2024-09-13T14:12:54.059+0000 7fd1d0408640 -1 osd.2 0 waiting for initial osdmap 2024-09-13T14:12:54.298 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[55591]: 2024-09-13T14:12:54.062+0000 7fd1c9a1a640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:12:54.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[39182]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:54.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[39182]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:54.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[39182]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:12:54.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:54.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[27770]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[27770]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[27770]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:12:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:54.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:54 smithi145 ceph-mon[31057]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:54 smithi145 ceph-mon[31057]: from='osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:12:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:54 smithi145 ceph-mon[31057]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:12:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048] boot 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048] boot 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:12:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:56.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:56.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: osd.2 [v2:172.21.15.26:6818/3032986048,v1:172.21.15.26:6819/3032986048] boot 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:12:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:12:57.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:57.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:57.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:57.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:57.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:57 smithi026 sudo[59575]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:12:57.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:57 smithi026 sudo[59575]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:12:57.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:57 smithi026 sudo[59575]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:12:57.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:57 smithi026 sudo[59575]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:12:58.378 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59690]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:12:58.378 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59690]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:12:58.378 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59690]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:12:58.378 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59690]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:12:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi026 2024-09-13T14:12:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 5356M 2024-09-13T14:12:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:12:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[27770]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[27770]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi026 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 5356M 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[39182]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:12:58.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:58 smithi026 ceph-mon[39182]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:12:58.384 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:12:58.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_1 2024-09-13T14:12:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:58 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi026 2024-09-13T14:12:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:58 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 5356M 2024-09-13T14:12:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:58 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:12:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:58 smithi145 ceph-mon[31057]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:12:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:58 smithi145 ceph-mon[31057]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:12:58.662 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:58.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59736]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:12:58.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59736]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:12:58.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59736]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:12:58.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:58 smithi026 sudo[59736]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:12:59.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:59 smithi026 ceph-mon[27770]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:12:59.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:12:59 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:59.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:59 smithi026 ceph-mon[39182]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:12:59.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:12:59 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:59.190 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59795]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:12:59.190 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59795]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:12:59.190 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59795]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:12:59.191 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59795]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:12:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:59 smithi145 ceph-mon[31057]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:12:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:12:59 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:12:59.843 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59814]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:12:59.843 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59814]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:12:59.843 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59814]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:12:59.843 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:12:59 smithi026 sudo[59814]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:00.123 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59840]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:13:00.124 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59840]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:00.124 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59840]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:00.124 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59840]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:00.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59889]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:13:00.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59889]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:00.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59889]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:00.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59889]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:01.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:00 smithi026 ceph-mon[39182]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:01.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:00 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59893]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59893]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59893]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 sudo[59893]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 ceph-mon[27770]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:00 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:00 smithi145 ceph-mon[31057]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:00 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33782]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33782]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33782]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:01.173 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33782]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33785]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:13:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33785]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33785]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:01 smithi145 sudo[33785]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:01 smithi026 sudo[60057]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:13:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:01 smithi026 sudo[60057]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:01 smithi026 sudo[60057]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:01 smithi026 sudo[60057]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:02.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:02.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 sudo[60068]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 sudo[60068]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 sudo[60068]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 sudo[60068]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:02.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:02 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:13:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:13:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:02 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:13:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2662280986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]': finished 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:03.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2662280986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]': finished 2024-09-13T14:13:03.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:13:03.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:03 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2662280986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]: dispatch 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fa5de3-4d3b-4555-ba58-e4ec6a10369d"}]': finished 2024-09-13T14:13:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:13:03.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:03 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:04.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:04 smithi145 ceph-mon[31057]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:04.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:04 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2039137835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:04 smithi026 ceph-mon[27770]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:04 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2039137835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:04 smithi026 ceph-mon[39182]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:04 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2039137835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:06 smithi026 ceph-mon[27770]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:06 smithi026 ceph-mon[39182]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:06 smithi145 ceph-mon[31057]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:08 smithi026 ceph-mon[27770]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T14:13:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:08 smithi026 ceph-mon[39182]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T14:13:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:08 smithi145 ceph-mon[31057]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[27770]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[39182]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:13:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:10 smithi145 ceph-mon[31057]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:13:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:13:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:11 smithi026 ceph-mon[27770]: Deploying daemon osd.3 on smithi026 2024-09-13T14:13:11.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:11 smithi026 ceph-mon[39182]: Deploying daemon osd.3 on smithi026 2024-09-13T14:13:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:11 smithi145 ceph-mon[31057]: Deploying daemon osd.3 on smithi026 2024-09-13T14:13:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:12 smithi026 ceph-mon[27770]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:13:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:12 smithi026 ceph-mon[39182]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:13:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:12 smithi145 ceph-mon[31057]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:13:14.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:14 smithi026 ceph-mon[27770]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:13:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:14 smithi026 ceph-mon[39182]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:13:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:14 smithi145 ceph-mon[31057]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:13:15.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:15.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:15.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:15.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:15.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:15.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:15 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:15 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:15 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:16.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:16 smithi026 ceph-mon[39182]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-09-13T14:13:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:16 smithi026 ceph-mon[27770]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-09-13T14:13:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:16 smithi145 ceph-mon[31057]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[27770]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[27770]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[27770]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[39182]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[39182]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:18.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:18 smithi026 ceph-mon[39182]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:18.493 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 3 on host 'smithi026' 2024-09-13T14:13:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:18 smithi145 ceph-mon[31057]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:13:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:18 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:18 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:18 smithi145 ceph-mon[31057]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:18 smithi145 ceph-mon[31057]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:13:19.161 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T14:13:19.163 INFO:tasks.cephadm:Deploying osd.4 on smithi145 with /dev/vg_nvme/lv_4... 2024-09-13T14:13:19.164 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:13:19.411 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[27770]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:13:19.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:13:19.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:19.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:19.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-mon[39182]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:19.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:19 smithi145 ceph-mon[31057]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-09-13T14:13:20.083 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[61949]: 2024-09-13T14:13:19.887+0000 7fe59c399640 -1 osd.3 0 waiting for initial osdmap 2024-09-13T14:13:20.083 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:13:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[61949]: 2024-09-13T14:13:19.890+0000 7fe5971d0640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:13:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:20 smithi145 ceph-mon[31057]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:20 smithi145 ceph-mon[31057]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:13:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:20 smithi145 ceph-mon[31057]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:13:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:20 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[27770]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[27770]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[27770]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[39182]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[39182]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[39182]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:13:20.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:20.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:20 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.772 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:13:21.772 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:13:21.772 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.772 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327] boot 2024-09-13T14:13:21.772 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:13:21.773 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:21 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.777 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:13:21.797 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_4 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327] boot 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: osd.3 [v2:172.21.15.26:6826/3933565327,v1:172.21.15.26:6827/3933565327] boot 2024-09-13T14:13:21.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:13:21.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:21 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:13:22.038 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:13:22.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:22.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:13:22.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:22.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:22.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:22 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:22 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:23.614 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi026 2024-09-13T14:13:23.614 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 4017M 2024-09-13T14:13:23.615 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:23.615 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:23.615 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:23.615 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:23 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi026 2024-09-13T14:13:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 4017M 2024-09-13T14:13:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi026 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 4017M 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:23 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[39182]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[27770]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:24 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:24.928 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:24 smithi145 ceph-mon[31057]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:24.929 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:24 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:24.929 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:24 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:24.929 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:24 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:25.825 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:25 smithi145 ceph-mon[31057]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:25.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:25 smithi026 ceph-mon[39182]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:25.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:25 smithi026 ceph-mon[27770]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:26.657 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:26.657 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/1925955930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.657 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.658 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]': finished 2024-09-13T14:13:26.658 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:13:26.658 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:26 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:26.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/1925955930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]': finished 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/1925955930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]: dispatch 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0c8497e-75fb-4238-9c1e-c1593ecf2507"}]': finished 2024-09-13T14:13:26.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:13:26.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:26 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:27.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:27 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/4201350475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:27.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:27 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/4201350475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:27 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/4201350475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:28.794 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:28 smithi145 ceph-mon[31057]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:28.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:28 smithi026 ceph-mon[39182]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:28.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:28 smithi026 ceph-mon[27770]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:30.777 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:30 smithi145 ceph-mon[31057]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:30.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:30 smithi026 ceph-mon[27770]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:30.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:30 smithi026 ceph-mon[39182]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:32.702 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:32 smithi145 ceph-mon[31057]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:32.702 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:32 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:13:32.702 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:32 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:32.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[39182]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:32.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:13:32.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:32.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[27770]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:32.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:13:32.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:32 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:33 smithi145 ceph-mon[31057]: Deploying daemon osd.4 on smithi145 2024-09-13T14:13:33.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:33 smithi026 ceph-mon[27770]: Deploying daemon osd.4 on smithi145 2024-09-13T14:13:33.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:33 smithi026 ceph-mon[39182]: Deploying daemon osd.4 on smithi145 2024-09-13T14:13:34.538 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:34 smithi145 ceph-mon[31057]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:34 smithi026 ceph-mon[27770]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:34 smithi026 ceph-mon[39182]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:36.568 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:36 smithi145 ceph-mon[31057]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:36 smithi026 ceph-mon[27770]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:36.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:36 smithi026 ceph-mon[39182]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:37.561 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:37.561 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:37.561 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:37.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:38.574 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:38 smithi145 ceph-mon[31057]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:38 smithi026 ceph-mon[27770]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:38.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:38 smithi026 ceph-mon[39182]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:40.193 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 4 on host 'smithi145' 2024-09-13T14:13:40.819 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.820 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:40 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.844 DEBUG:teuthology.orchestra.run.smithi145:osd.4> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T14:13:40.846 INFO:tasks.cephadm:Deploying osd.5 on smithi145 with /dev/vg_nvme/lv_3... 2024-09-13T14:13:40.847 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:13:40.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:40.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:40 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:41.578 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-mon[31057]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:41.956 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[36471]: 2024-09-13T14:13:41.832+0000 7fa375fb0640 -1 osd.4 0 waiting for initial osdmap 2024-09-13T14:13:41.956 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:13:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[36471]: 2024-09-13T14:13:41.837+0000 7fa36f5e4640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:13:42.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[27770]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: from='osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:42.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:13:42.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:41 smithi026 ceph-mon[39182]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[39182]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[39182]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[27770]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[27770]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:43.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:42 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:42 smithi145 ceph-mon[31057]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:13:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:42 smithi145 ceph-mon[31057]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:13:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:42 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:42 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871] boot 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:44.086 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:43 smithi145 ceph-mon[31057]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:13:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:13:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:13:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871] boot 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[27770]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: osd.4 [v2:172.21.15.145:6800/2159398871,v1:172.21.15.145:6801/2159398871] boot 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:44.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:43 smithi026 ceph-mon[39182]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:13:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:13:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:45.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:45.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:45.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.018 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:13:46.041 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_3 2024-09-13T14:13:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:13:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:13:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:46.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:46.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:13:46.279 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:13:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:46 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi145 2024-09-13T14:13:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:46 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 17088M 2024-09-13T14:13:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:46 smithi145 ceph-mon[31057]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi145 2024-09-13T14:13:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 17088M 2024-09-13T14:13:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[39182]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi145 2024-09-13T14:13:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 17088M 2024-09-13T14:13:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:46 smithi026 ceph-mon[27770]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:48 smithi145 ceph-mon[31057]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-09-13T14:13:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:48 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:48 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:48 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[27770]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-09-13T14:13:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[39182]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-09-13T14:13:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:48 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:49.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:49 smithi145 ceph-mon[31057]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:50.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:49 smithi026 ceph-mon[27770]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:50.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:49 smithi026 ceph-mon[39182]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:50.844 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:13:50.844 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/2605574830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:50.844 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:50.845 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]': finished 2024-09-13T14:13:50.845 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: osdmap e33: 6 total, 5 up, 6 in 2024-09-13T14:13:50.845 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:50 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:13:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:13:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/2605574830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]': finished 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: osdmap e33: 6 total, 5 up, 6 in 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/2605574830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]: dispatch 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa1efee7-55a1-4905-a6ca-b3c54deb71bd"}]': finished 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: osdmap e33: 6 total, 5 up, 6 in 2024-09-13T14:13:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:50 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:13:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:51 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/2258029136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:51 smithi145 ceph-mon[31057]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-13T14:13:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:51 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/2258029136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:51 smithi026 ceph-mon[27770]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-13T14:13:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:51 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/2258029136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:51 smithi026 ceph-mon[39182]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-13T14:13:54.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:54 smithi026 ceph-mon[27770]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:13:54.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:54 smithi026 ceph-mon[39182]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:13:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:54 smithi145 ceph-mon[31057]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:13:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:56 smithi026 ceph-mon[27770]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:13:56.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:56 smithi026 ceph-mon[39182]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:13:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:56 smithi145 ceph-mon[31057]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:13:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:13:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:57 smithi145 ceph-mon[31057]: Deploying daemon osd.5 on smithi145 2024-09-13T14:13:57.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:13:57.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:57.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[27770]: Deploying daemon osd.5 on smithi145 2024-09-13T14:13:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:13:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:57 smithi026 ceph-mon[39182]: Deploying daemon osd.5 on smithi145 2024-09-13T14:13:58.358 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:13:58 smithi145 ceph-mon[31057]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:58.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:13:58 smithi026 ceph-mon[39182]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:13:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:13:58 smithi026 ceph-mon[27770]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:00 smithi145 ceph-mon[31057]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:00 smithi026 ceph-mon[27770]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:00 smithi026 ceph-mon[39182]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:01 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:01 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:01 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:01 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:02.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:02 smithi026 ceph-mon[27770]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:02 smithi026 ceph-mon[39182]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:02.697 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:02 smithi145 ceph-mon[31057]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:04.524 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:04 smithi145 ceph-mon[31057]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:04.558 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 5 on host 'smithi145' 2024-09-13T14:14:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:04 smithi026 ceph-mon[27770]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:04.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:04 smithi026 ceph-mon[39182]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:05.201 DEBUG:teuthology.orchestra.run.smithi145:osd.5> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:14:05.203 INFO:tasks.cephadm:Deploying osd.6 on smithi145 with /dev/vg_nvme/lv_2... 2024-09-13T14:14:05.203 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:14:05.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:05 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.505 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:14:05.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:05.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:05 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:14:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:14:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[27770]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:06 smithi026 ceph-mon[39182]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:14:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:14:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: from='osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-mon[31057]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:06.706 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[42486]: 2024-09-13T14:14:06.320+0000 7efe95bd9640 -1 osd.5 0 waiting for initial osdmap 2024-09-13T14:14:06.706 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:14:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[42486]: 2024-09-13T14:14:06.326+0000 7efe90a10640 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:14:07.488 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:07 smithi145 ceph-mon[31057]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:07.488 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:07 smithi145 ceph-mon[31057]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:14:07.488 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:07 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:07.488 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:07 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[27770]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[27770]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:14:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:07.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[39182]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[39182]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:14:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:07 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839] boot 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: osdmap e36: 6 total, 6 up, 6 in 2024-09-13T14:14:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:08 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:14:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:14:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839] boot 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: osdmap e36: 6 total, 6 up, 6 in 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: osd.5 [v2:172.21.15.145:6808/2344369839,v1:172.21.15.145:6809/2344369839] boot 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: osdmap e36: 6 total, 6 up, 6 in 2024-09-13T14:14:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:08 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:14:09.245 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:14:09.265 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_2 2024-09-13T14:14:09.550 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:09 smithi145 ceph-mon[31057]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:14:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:09 smithi026 ceph-mon[27770]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:14:09.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:09 smithi026 ceph-mon[39182]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:14:09.825 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:14:10.582 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:10 smithi145 ceph-mon[31057]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:10.583 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:10 smithi145 ceph-mon[31057]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:14:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:10 smithi026 ceph-mon[27770]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:10 smithi026 ceph-mon[27770]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:14:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:10 smithi026 ceph-mon[39182]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:10 smithi026 ceph-mon[39182]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:14:12.590 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:12.590 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.590 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:12.591 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:12.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:13.629 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:13 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi145 2024-09-13T14:14:13.629 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:13 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 8544M 2024-09-13T14:14:13.629 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:13 smithi145 ceph-mon[31057]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:13.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi145 2024-09-13T14:14:13.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 8544M 2024-09-13T14:14:13.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[39182]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi145 2024-09-13T14:14:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 8544M 2024-09-13T14:14:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:13 smithi026 ceph-mon[27770]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:14.602 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:14 smithi145 ceph-mon[31057]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:14.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:14 smithi026 ceph-mon[27770]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:14 smithi026 ceph-mon[39182]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:15.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/1970234679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]': finished 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: osdmap e39: 7 total, 6 up, 7 in 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/925872571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/1970234679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]': finished 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: osdmap e39: 7 total, 6 up, 7 in 2024-09-13T14:14:15.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:15 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/925872571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/1970234679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]: dispatch 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79482f04-e0b2-494c-8d25-ab2ff68d191c"}]': finished 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: osdmap e39: 7 total, 6 up, 7 in 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:15 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/925872571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:16 smithi026 ceph-mon[27770]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:16 smithi026 ceph-mon[39182]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:16 smithi145 ceph-mon[31057]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:18.432 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:18 smithi145 ceph-mon[31057]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:18.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:18 smithi026 ceph-mon[39182]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:18 smithi026 ceph-mon[27770]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:20.417 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:20 smithi145 ceph-mon[31057]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:20.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:20 smithi026 ceph-mon[39182]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:20 smithi026 ceph-mon[27770]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:21.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:14:21.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:21.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[39182]: Deploying daemon osd.6 on smithi145 2024-09-13T14:14:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:14:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:21 smithi026 ceph-mon[27770]: Deploying daemon osd.6 on smithi145 2024-09-13T14:14:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:21 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:14:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:21 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:21 smithi145 ceph-mon[31057]: Deploying daemon osd.6 on smithi145 2024-09-13T14:14:22.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:22 smithi026 ceph-mon[39182]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:22.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:22 smithi026 ceph-mon[27770]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:22 smithi145 ceph-mon[31057]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:24 smithi145 ceph-mon[31057]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:24.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:24 smithi026 ceph-mon[27770]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:24.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:24 smithi026 ceph-mon[39182]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:26.637 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:26 smithi145 ceph-mon[31057]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:26.638 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:26 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:26.638 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:26 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:26.638 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:26 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:26.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[27770]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[39182]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:26 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:28 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:28 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:28.826 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 6 on host 'smithi145' 2024-09-13T14:14:29.442 DEBUG:teuthology.orchestra.run.smithi145:osd.6> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T14:14:29.445 INFO:tasks.cephadm:Deploying osd.7 on smithi145 with /dev/vg_nvme/lv_1... 2024-09-13T14:14:29.445 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:14:29.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[27770]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[39182]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:29 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:29.677 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:29 smithi145 ceph-mon[31057]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:14:29.678 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:29.678 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:29.678 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:29 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:30.246 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:14:30.585 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-mon[31057]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:30.585 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-mon[31057]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:14:30.585 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-mon[31057]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:14:30.585 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:30.585 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-mon[31057]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:30.585 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[48379]: 2024-09-13T14:14:30.398+0000 7fcee61aa640 -1 osd.6 0 waiting for initial osdmap 2024-09-13T14:14:30.585 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:14:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[48379]: 2024-09-13T14:14:30.406+0000 7fcee17e2640 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:14:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[27770]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[27770]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[27770]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[27770]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[39182]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[39182]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[39182]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:30 smithi026 ceph-mon[39182]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:31.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:31 smithi145 ceph-mon[31057]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:31.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:31 smithi145 ceph-mon[31057]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:14:31.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:31 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:31.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:31 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[27770]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[27770]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:14:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[39182]: from='osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[39182]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:14:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:31 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777] boot 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: osdmap e42: 7 total, 7 up, 7 in 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:14:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:32.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777] boot 2024-09-13T14:14:32.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: osdmap e42: 7 total, 7 up, 7 in 2024-09-13T14:14:32.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:32 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: osd.6 [v2:172.21.15.145:6816/913657777,v1:172.21.15.145:6817/913657777] boot 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: osdmap e42: 7 total, 7 up, 7 in 2024-09-13T14:14:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:32 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:14:33.539 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:14:33.539 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.539 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.539 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.540 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:33 smithi145 ceph-mon[31057]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:14:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:14:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[27770]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:33.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:33 smithi026 ceph-mon[39182]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:14:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:34 smithi145 ceph-mon[31057]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:34 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi145 2024-09-13T14:14:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:34 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 5696M 2024-09-13T14:14:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:34 smithi145 ceph-mon[31057]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:34.778 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-09-13T14:14:34.805 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_1 2024-09-13T14:14:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[27770]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi145 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 5696M 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[27770]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[39182]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi145 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 5696M 2024-09-13T14:14:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:34 smithi026 ceph-mon[39182]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:35.054 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:14:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:36 smithi145 ceph-mon[31057]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:36 smithi026 ceph-mon[27770]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:36.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:36 smithi026 ceph-mon[39182]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:37.441 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:37.442 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:37.442 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:37 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:37.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:37.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:37.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:37 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:38.523 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:38 smithi145 ceph-mon[31057]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:38.524 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:38 smithi145 ceph-mon[31057]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:38 smithi026 ceph-mon[27770]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:38 smithi026 ceph-mon[27770]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:38 smithi026 ceph-mon[39182]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:38 smithi026 ceph-mon[39182]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:39.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:39 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/519746780' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:39 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:39 smithi145 ceph-mon[31057]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]': finished 2024-09-13T14:14:39.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:39 smithi145 ceph-mon[31057]: osdmap e45: 8 total, 7 up, 8 in 2024-09-13T14:14:39.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:39 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/519746780' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[27770]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]': finished 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[27770]: osdmap e45: 8 total, 7 up, 8 in 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/519746780' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]: dispatch 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[39182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f406363-f9fd-4e98-b256-0fb5b4ffaf60"}]': finished 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[39182]: osdmap e45: 8 total, 7 up, 8 in 2024-09-13T14:14:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:39 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:40 smithi145 ceph-mon[31057]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:14:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:40 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/2081072940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:40 smithi145 ceph-mon[31057]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:14:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:40 smithi145 ceph-mon[31057]: Cluster is now healthy 2024-09-13T14:14:40.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[39182]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:14:40.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/2081072940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:40.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[39182]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:14:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[39182]: Cluster is now healthy 2024-09-13T14:14:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[27770]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:14:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/2081072940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[27770]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:14:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:40 smithi026 ceph-mon[27770]: Cluster is now healthy 2024-09-13T14:14:42.495 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:42 smithi145 ceph-mon[31057]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:14:42.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:42 smithi026 ceph-mon[39182]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:14:42.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:42 smithi026 ceph-mon[27770]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T14:14:44.530 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:44 smithi145 ceph-mon[31057]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:14:44.718 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:44 smithi026 ceph-mon[27770]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:14:44.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:44 smithi026 ceph-mon[39182]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T14:14:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:14:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:45.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:14:45.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:14:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:46.438 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:46 smithi145 ceph-mon[31057]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:46.438 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:46 smithi145 ceph-mon[31057]: Deploying daemon osd.7 on smithi145 2024-09-13T14:14:46.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:46 smithi026 ceph-mon[27770]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:46.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:46 smithi026 ceph-mon[27770]: Deploying daemon osd.7 on smithi145 2024-09-13T14:14:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:46 smithi026 ceph-mon[39182]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:46 smithi026 ceph-mon[39182]: Deploying daemon osd.7 on smithi145 2024-09-13T14:14:48.608 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:48 smithi145 ceph-mon[31057]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:48 smithi026 ceph-mon[27770]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:48.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:48 smithi026 ceph-mon[39182]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:14:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:50 smithi145 ceph-mon[31057]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:50 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:50 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:50 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:50.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[27770]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:50.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:50.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[39182]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:50 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:52 smithi145 ceph-mon[31057]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:52 smithi026 ceph-mon[27770]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:52 smithi026 ceph-mon[39182]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:53 smithi145 ceph-mon[31057]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:53 smithi145 ceph-mon[31057]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:53.552 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 7 on host 'smithi145' 2024-09-13T14:14:53.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:53 smithi026 ceph-mon[27770]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:53.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:53 smithi026 ceph-mon[27770]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:53.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:53 smithi026 ceph-mon[39182]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:53.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:53 smithi026 ceph-mon[39182]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:14:54.121 DEBUG:teuthology.orchestra.run.smithi145:osd.7> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T14:14:54.123 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T14:14:54.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:14:54.390 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:54.722 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.723 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.724 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.724 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.724 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.724 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[54871]: 2024-09-13T14:14:54.457+0000 7fcae6b40640 -1 osd.7 0 waiting for initial osdmap 2024-09-13T14:14:54.725 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:14:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[54871]: 2024-09-13T14:14:54.464+0000 7fcae0953640 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:54.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:54 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:55.553 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:55 smithi145 ceph-mon[31057]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:55.554 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:55 smithi145 ceph-mon[31057]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:14:55.554 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:55 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:55.554 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:55 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:55.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[27770]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[27770]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[39182]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[39182]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:55 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.503 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:14:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: purged_snaps scrub starts 2024-09-13T14:14:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: purged_snaps scrub ok 2024-09-13T14:14:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206] boot 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: osdmap e48: 8 total, 8 up, 8 in 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: purged_snaps scrub starts 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: purged_snaps scrub ok 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206] boot 2024-09-13T14:14:56.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: osdmap e48: 8 total, 8 up, 8 in 2024-09-13T14:14:56.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.884 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: purged_snaps scrub starts 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: purged_snaps scrub ok 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: osd.7 [v2:172.21.15.145:6824/2626541206,v1:172.21.15.145:6825/2626541206] boot 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: osdmap e48: 8 total, 8 up, 8 in 2024-09-13T14:14:56.885 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:14:57.248 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1726236895,"num_in_osds":8,"osd_in_since":1726236879,"num_remapped_pgs":0} 2024-09-13T14:14:57.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:14:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/354105074' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/354105074' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:14:57.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/354105074' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:14:57.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:57.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:14:58.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[27770]: Detected new or changed devices on smithi145 2024-09-13T14:14:58.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 4272M 2024-09-13T14:14:58.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[27770]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[27770]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:14:58.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[27770]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[39182]: Detected new or changed devices on smithi145 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 4272M 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[39182]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[39182]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:14:58 smithi026 ceph-mon[39182]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:58 smithi145 ceph-mon[31057]: Detected new or changed devices on smithi145 2024-09-13T14:14:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:58 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 4272M 2024-09-13T14:14:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:58 smithi145 ceph-mon[31057]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:58 smithi145 ceph-mon[31057]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:14:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:14:58 smithi145 ceph-mon[31057]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:14:59.567 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:14:59.568 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":50,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:14:57.463942+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:15:00.374 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T14:12:55.698911+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T14:15:00.374 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T14:15:00.634 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:00 smithi026 ceph-mon[27770]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:00 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/982365837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:00.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:00 smithi026 ceph-mon[39182]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:00.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:00 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/982365837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:00 smithi145 ceph-mon[31057]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:00 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/982365837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:02.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:02 smithi026 ceph-mon[27770]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:02.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:02 smithi026 ceph-mon[39182]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:02.720 INFO:teuthology.orchestra.run.smithi026.stdout:pg_num: 1 2024-09-13T14:15:02.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:02 smithi145 ceph-mon[31057]: pgmap v168: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:03.333 INFO:tasks.cephadm:Adding mds.a on smithi026 2024-09-13T14:15:03.333 INFO:tasks.cephadm:Adding mds.b on smithi145 2024-09-13T14:15:03.333 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mds all '2;smithi026=a;smithi145=b' 2024-09-13T14:15:03.590 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:03 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3645664121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:15:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:03 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3645664121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:15:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:03 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3645664121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:15:04.800 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:04 smithi145 ceph-mon[31057]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:04.800 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:04 smithi145 ceph-mon[31057]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:15:04.800 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:04 smithi145 ceph-mon[31057]: Cluster is now healthy 2024-09-13T14:15:04.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[27770]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:04.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[27770]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:15:04.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[27770]: Cluster is now healthy 2024-09-13T14:15:04.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[39182]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:04.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[39182]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:15:04.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:04 smithi026 ceph-mon[39182]: Cluster is now healthy 2024-09-13T14:15:05.451 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled mds.all update... 2024-09-13T14:15:06.047 DEBUG:teuthology.orchestra.run.smithi026:mds.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.a.service 2024-09-13T14:15:06.050 DEBUG:teuthology.orchestra.run.smithi145:mds.b> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.b.service 2024-09-13T14:15:06.052 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-13T14:15:06.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:15:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T14:15:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: Saving service mds.all spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[27770]: Deploying daemon mds.a on smithi026 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: Saving service mds.all spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:06 smithi026 ceph-mon[39182]: Deploying daemon mds.a on smithi026 2024-09-13T14:15:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: Saving service mds.all spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:06 smithi145 ceph-mon[31057]: Deploying daemon mds.a on smithi026 2024-09-13T14:15:07.280 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:08 smithi145 ceph-mon[31057]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:15:08.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:08 smithi026 ceph-mon[27770]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:15:08.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:08 smithi026 ceph-mon[39182]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:15:09.336 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:09.337 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-09-13T14:15:09.337 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 1 2024-09-13T14:15:09.337 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 podman[67025]: 2024-09-13 14:15:09.215641567 +0000 UTC m=+0.289941424 container create 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:15:09.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:09 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/4152141194' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:09.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:09 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4152141194' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:09.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:09 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/4152141194' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:10.097 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 podman[67025]: 2024-09-13 14:15:09.803389571 +0000 UTC m=+0.877689428 container init 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:15:10.098 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 podman[67025]: 2024-09-13 14:15:09.808944397 +0000 UTC m=+0.883244256 container start 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:15:10.098 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[67082]: starting mds.a at 2024-09-13T14:15:10.098 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 bash[67025]: 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea 2024-09-13T14:15:10.098 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:15:09 smithi026 systemd[1]: Started Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:15:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:15:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: mds.? [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] up:boot 2024-09-13T14:15:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: fsmap 1 up:standby 2024-09-13T14:15:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:10.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: mds.? [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] up:boot 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: fsmap 1 up:standby 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:10.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: mds.? [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] up:boot 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: fsmap 1 up:standby 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:15:10.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:11 smithi145 ceph-mon[31057]: Deploying daemon mds.b on smithi145 2024-09-13T14:15:11.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:11 smithi026 ceph-mon[27770]: Deploying daemon mds.b on smithi145 2024-09-13T14:15:11.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:11 smithi026 ceph-mon[39182]: Deploying daemon mds.b on smithi145 2024-09-13T14:15:12.085 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:11 smithi145 podman[58897]: 2024-09-13 14:15:11.980789197 +0000 UTC m=+0.123100315 container create 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:15:12.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:12 smithi145 podman[58897]: 2024-09-13 14:15:12.183103258 +0000 UTC m=+0.325414331 container init 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T14:15:12.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:12 smithi145 podman[58897]: 2024-09-13 14:15:12.188055138 +0000 UTC m=+0.330366211 container start 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:15:12.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[58912]: starting mds.b at 2024-09-13T14:15:12.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:12 smithi145 bash[58897]: 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 2024-09-13T14:15:12.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:12 smithi145 systemd[1]: Started Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:15:12.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:15:12.765 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:12.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:13.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[27770]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:boot 2024-09-13T14:15:13.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[27770]: fsmap 2 up:standby 2024-09-13T14:15:13.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:15:13.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[39182]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:boot 2024-09-13T14:15:13.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[39182]: fsmap 2 up:standby 2024-09-13T14:15:13.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:13 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:15:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:13 smithi145 ceph-mon[31057]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:boot 2024-09-13T14:15:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:13 smithi145 ceph-mon[31057]: fsmap 2 up:standby 2024-09-13T14:15:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:13 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:15:14.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[39182]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:14.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:14.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:14.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[27770]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:14.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:14.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:14 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:14.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:14 smithi145 ceph-mon[31057]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:15:14.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:14 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:14.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:14 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:15.339 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:15.340 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14421,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.26:6835/248064250","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":248064250},{"type":"v1","addr":"172.21.15.26:6835","nonce":248064250}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24331,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.145:6833/2983197144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2983197144},{"type":"v1","addr":"172.21.15.145:6833","nonce":2983197144}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2024-09-13T14:15:15.340 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 3 2024-09-13T14:15:15.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph --format=json-pretty osd lspools 2024-09-13T14:15:16.284 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2363673328' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2363673328' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:16 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2363673328' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:16 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout:[ 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "poolnum": 1, 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "poolname": ".mgr" 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:15:18.201 INFO:teuthology.orchestra.run.smithi026.stdout:] 2024-09-13T14:15:18.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:18 smithi026 ceph-mon[27770]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:18.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:18 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/560611338' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-13T14:15:18.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:18 smithi026 ceph-mon[39182]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:18.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:18 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/560611338' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-13T14:15:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:18 smithi145 ceph-mon[31057]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:18 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/560611338' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-13T14:15:18.807 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-13T14:15:18.807 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-13T14:15:19.062 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:20.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:20 smithi026 ceph-mon[27770]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:20.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:20 smithi026 ceph-mon[39182]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:20 smithi145 ceph-mon[31057]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:21.534 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'cephfs_metadata' created 2024-09-13T14:15:21.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:21 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3781248879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:21.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:21 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:21 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3781248879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:21 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:21 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3781248879' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:21 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-13T14:15:22.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-13T14:15:22.661 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[27770]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-13T14:15:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[27770]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:15:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[27770]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:15:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[39182]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-13T14:15:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[39182]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:15:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:22 smithi026 ceph-mon[39182]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:15:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:22 smithi145 ceph-mon[31057]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:22 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-13T14:15:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:22 smithi145 ceph-mon[31057]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:15:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:22 smithi145 ceph-mon[31057]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:15:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[39182]: pgmap v181: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[39182]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:15:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[39182]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[27770]: pgmap v181: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[27770]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:15:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:24 smithi026 ceph-mon[27770]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:24 smithi145 ceph-mon[31057]: pgmap v181: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:24 smithi145 ceph-mon[31057]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:15:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:24 smithi145 ceph-mon[31057]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:25.546 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'cephfs_data' created 2024-09-13T14:15:25.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:25 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2183582372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:25.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:25 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:25.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:25 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2183582372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:25.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:25 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:25 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2183582372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:25 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-13T14:15:26.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-13T14:15:26.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[27770]: pgmap v183: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-13T14:15:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[27770]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:15:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[39182]: pgmap v183: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-13T14:15:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:26 smithi026 ceph-mon[39182]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:15:26.719 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:26 smithi145 ceph-mon[31057]: pgmap v183: 33 pgs: 13 creating+peering, 18 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:26 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-13T14:15:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:26 smithi145 ceph-mon[31057]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:15:27.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:27 smithi026 ceph-mon[27770]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:15:27.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:27 smithi026 ceph-mon[39182]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:15:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:27 smithi145 ceph-mon[31057]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:15:28.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:28 smithi026 ceph-mon[27770]: pgmap v186: 97 pgs: 39 creating+peering, 24 unknown, 34 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:28.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:28 smithi026 ceph-mon[27770]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:15:28.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:28 smithi026 ceph-mon[39182]: pgmap v186: 97 pgs: 39 creating+peering, 24 unknown, 34 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:28.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:28 smithi026 ceph-mon[39182]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:15:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:28 smithi145 ceph-mon[31057]: pgmap v186: 97 pgs: 39 creating+peering, 24 unknown, 34 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:28 smithi145 ceph-mon[31057]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:15:29.561 INFO:teuthology.orchestra.run.smithi026.stderr:new fs with metadata pool 2 and data pool 3 2024-09-13T14:15:29.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-13T14:15:29.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[39182]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-13T14:15:29.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[39182]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:29.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-13T14:15:29.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[27770]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-13T14:15:29.899 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:29 smithi026 ceph-mon[27770]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:29 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-13T14:15:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:29 smithi145 ceph-mon[31057]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-13T14:15:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:29 smithi145 ceph-mon[31057]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:15:30.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:15:30.524 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[39182]: pgmap v188: 97 pgs: 29 creating+peering, 68 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[39182]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[39182]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[27770]: pgmap v188: 97 pgs: 29 creating+peering, 68 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[27770]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-13T14:15:30.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[27770]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-13T14:15:30.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:30 smithi026 ceph-mon[27770]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-13T14:15:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:30 smithi145 ceph-mon[31057]: pgmap v188: 97 pgs: 29 creating+peering, 68 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:30 smithi145 ceph-mon[31057]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:15:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:30 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3665926772' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-13T14:15:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:30 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-13T14:15:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:30 smithi145 ceph-mon[31057]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-13T14:15:31.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[39182]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:active 2024-09-13T14:15:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[27770]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:active 2024-09-13T14:15:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[27770]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:31 smithi026 ceph-mon[27770]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:31 smithi145 ceph-mon[31057]: mds.? [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] up:active 2024-09-13T14:15:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:31 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:31 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:32.449 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:32.449 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":57,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:15:28.665306+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:15:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[27770]: pgmap v190: 97 pgs: 27 creating+peering, 70 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-13T14:15:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[27770]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-13T14:15:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[27770]: Cluster is now healthy 2024-09-13T14:15:32.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3604731141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[39182]: pgmap v190: 97 pgs: 27 creating+peering, 70 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-13T14:15:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[39182]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-13T14:15:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[39182]: Cluster is now healthy 2024-09-13T14:15:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:32 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3604731141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:32.955 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:15:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[58912]: 2024-09-13T14:15:32.470+0000 7fcc71033640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-13T14:15:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:32 smithi145 ceph-mon[31057]: pgmap v190: 97 pgs: 27 creating+peering, 70 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-13T14:15:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:32 smithi145 ceph-mon[31057]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-13T14:15:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:32 smithi145 ceph-mon[31057]: Cluster is now healthy 2024-09-13T14:15:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:32 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3604731141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:33.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:15:33.306 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:34.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:34 smithi026 ceph-mon[39182]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-09-13T14:15:34.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:34 smithi026 ceph-mon[27770]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-09-13T14:15:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:34 smithi145 ceph-mon[31057]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-09-13T14:15:35.148 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:35.149 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":57,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:15:28.665306+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:15:35.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:35 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3778622367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:35 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3778622367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:35.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs set cephfs standby_count_wanted 0 2024-09-13T14:15:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:35 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3778622367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:36.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:36 smithi026 ceph-mon[27770]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:15:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:36 smithi026 ceph-mon[39182]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:15:36.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:36 smithi145 ceph-mon[31057]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-13T14:15:38.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:38 smithi026 ceph-mon[39182]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:38.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:38 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-13T14:15:38.891 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:38 smithi026 ceph-mon[27770]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:38.891 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:38 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-13T14:15:38.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:38 smithi145 ceph-mon[31057]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:38.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:38 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-13T14:15:39.314 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-13T14:15:39.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs set cephfs allow_standby_replay false 2024-09-13T14:15:39.574 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:39 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-13T14:15:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:39 smithi026 ceph-mon[27770]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:39 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-13T14:15:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:39 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:39 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/44178216' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-13T14:15:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:39 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:40.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:40 smithi026 ceph-mon[27770]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:40.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:40 smithi026 ceph-mon[39182]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:40.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:40 smithi145 ceph-mon[31057]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T14:15:41.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:41 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2105717701' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:41.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:41 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:41.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:41 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2105717701' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:41.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:41 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:41 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2105717701' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:41 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-13T14:15:42.330 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-13T14:15:42.330 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:15:42.595 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:42.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[39182]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:15:42.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-13T14:15:42.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:42.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[27770]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:15:42.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-13T14:15:42.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:42 smithi026 ceph-mon[27770]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:42 smithi145 ceph-mon[31057]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:15:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:42 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-13T14:15:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:42 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:15:44.537 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:44.537 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14421,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.26:6835/248064250","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":248064250},{"type":"v1","addr":"172.21.15.26:6835","nonce":248064250}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:15:41.581255+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24331},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24331":{"gid":24331,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.145:6833/2983197144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2983197144},{"type":"v1","addr":"172.21.15.145:6833","nonce":2983197144}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-13T14:15:44.538 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 8 2024-09-13T14:15:44.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:44 smithi026 ceph-mon[27770]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-13T14:15:44.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:44 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2148128964' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:44.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:44 smithi026 ceph-mon[39182]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-13T14:15:44.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:44 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2148128964' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:44 smithi145 ceph-mon[31057]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-13T14:15:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:44 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2148128964' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:15:45.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:15:45.477 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:45.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-13T14:15:45.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]: dispatch 2024-09-13T14:15:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-13T14:15:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:45 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-13T14:15:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:45 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]: dispatch 2024-09-13T14:15:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:15:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-13T14:15:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[27770]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-09-13T14:15:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:15:46.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2024-09-13T14:15:46.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-13T14:15:46.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-13T14:15:46.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:46 smithi026 ceph-mon[39182]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 1]}]': finished 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [0, 2]}]': finished 2024-09-13T14:15:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-13T14:15:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.30", "id": [0, 1]}]': finished 2024-09-13T14:15:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:46 smithi145 ceph-mon[31057]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:15:47.432 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:47.433 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":59,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:15:46.595491+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:15:47.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:47 smithi026 ceph-mon[39182]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:15:47.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:47 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3841225262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:47.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:47 smithi026 ceph-mon[27770]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:15:47.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:47 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3841225262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:47 smithi145 ceph-mon[31057]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:15:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:47 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3841225262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:15:48.190 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:15:48.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:15:48.451 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:48.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:48 smithi026 ceph-mon[27770]: pgmap v200: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:48.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:48 smithi026 ceph-mon[39182]: pgmap v200: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:48.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:48 smithi145 ceph-mon[31057]: pgmap v200: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:50.390 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:50.392 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-09-13T14:15:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:50 smithi026 ceph-mon[27770]: pgmap v201: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:50 smithi026 ceph-mon[39182]: pgmap v201: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:50 smithi145 ceph-mon[31057]: pgmap v201: 97 pgs: 1 activating, 5 peering, 91 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:51.142 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-13T14:15:49.261139+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385192,"kb_used_data":6340,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576024,"statfs":{"total":767960285184,"available":765517848576,"internally_reserved":0,"allocated":6492160,"data_stored":3739163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001674"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566011+0000","last_change":"2024-09-13T14:15:26.568393+0000","last_active":"2024-09-13T14:15:29.566011+0000","last_peered":"2024-09-13T14:15:29.566011+0000","last_clean":"2024-09-13T14:15:29.566011+0000","last_became_active":"2024-09-13T14:15:26.568075+0000","last_became_peered":"2024-09-13T14:15:26.568075+0000","last_unstale":"2024-09-13T14:15:29.566011+0000","last_undegraded":"2024-09-13T14:15:29.566011+0000","last_fullsized":"2024-09-13T14:15:29.566011+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567016+0000","last_change":"2024-09-13T14:15:26.563624+0000","last_active":"2024-09-13T14:15:29.567016+0000","last_peered":"2024-09-13T14:15:29.567016+0000","last_clean":"2024-09-13T14:15:29.567016+0000","last_became_active":"2024-09-13T14:15:26.563364+0000","last_became_peered":"2024-09-13T14:15:26.563364+0000","last_unstale":"2024-09-13T14:15:29.567016+0000","last_undegraded":"2024-09-13T14:15:29.567016+0000","last_fullsized":"2024-09-13T14:15:29.567016+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565402+0000","last_change":"2024-09-13T14:15:26.566586+0000","last_active":"2024-09-13T14:15:29.565402+0000","last_peered":"2024-09-13T14:15:29.565402+0000","last_clean":"2024-09-13T14:15:29.565402+0000","last_became_active":"2024-09-13T14:15:26.566331+0000","last_became_peered":"2024-09-13T14:15:26.566331+0000","last_unstale":"2024-09-13T14:15:29.565402+0000","last_undegraded":"2024-09-13T14:15:29.565402+0000","last_fullsized":"2024-09-13T14:15:29.565402+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601352+0000","last_change":"2024-09-13T14:15:26.564769+0000","last_active":"2024-09-13T14:15:45.601352+0000","last_peered":"2024-09-13T14:15:45.601352+0000","last_clean":"2024-09-13T14:15:45.601352+0000","last_became_active":"2024-09-13T14:15:26.564496+0000","last_became_peered":"2024-09-13T14:15:26.564496+0000","last_unstale":"2024-09-13T14:15:45.601352+0000","last_undegraded":"2024-09-13T14:15:45.601352+0000","last_fullsized":"2024-09-13T14:15:45.601352+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567518+0000","last_change":"2024-09-13T14:15:26.555592+0000","last_active":"2024-09-13T14:15:29.567518+0000","last_peered":"2024-09-13T14:15:29.567518+0000","last_clean":"2024-09-13T14:15:29.567518+0000","last_became_active":"2024-09-13T14:15:26.555311+0000","last_became_peered":"2024-09-13T14:15:26.555311+0000","last_unstale":"2024-09-13T14:15:29.567518+0000","last_undegraded":"2024-09-13T14:15:29.567518+0000","last_fullsized":"2024-09-13T14:15:29.567518+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601228+0000","last_change":"2024-09-13T14:15:26.572204+0000","last_active":"2024-09-13T14:15:45.601228+0000","last_peered":"2024-09-13T14:15:45.601228+0000","last_clean":"2024-09-13T14:15:45.601228+0000","last_became_active":"2024-09-13T14:15:26.571310+0000","last_became_peered":"2024-09-13T14:15:26.571310+0000","last_unstale":"2024-09-13T14:15:45.601228+0000","last_undegraded":"2024-09-13T14:15:45.601228+0000","last_fullsized":"2024-09-13T14:15:45.601228+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565271+0000","last_change":"2024-09-13T14:15:26.566676+0000","last_active":"2024-09-13T14:15:29.565271+0000","last_peered":"2024-09-13T14:15:29.565271+0000","last_clean":"2024-09-13T14:15:29.565271+0000","last_became_active":"2024-09-13T14:15:26.566403+0000","last_became_peered":"2024-09-13T14:15:26.566403+0000","last_unstale":"2024-09-13T14:15:29.565271+0000","last_undegraded":"2024-09-13T14:15:29.565271+0000","last_fullsized":"2024-09-13T14:15:29.565271+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596840+0000","last_change":"2024-09-13T14:15:26.566663+0000","last_active":"2024-09-13T14:15:45.596840+0000","last_peered":"2024-09-13T14:15:45.596840+0000","last_clean":"2024-09-13T14:15:45.596840+0000","last_became_active":"2024-09-13T14:15:26.566400+0000","last_became_peered":"2024-09-13T14:15:26.566400+0000","last_unstale":"2024-09-13T14:15:45.596840+0000","last_undegraded":"2024-09-13T14:15:45.596840+0000","last_fullsized":"2024-09-13T14:15:45.596840+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567714+0000","last_change":"2024-09-13T14:15:26.570338+0000","last_active":"2024-09-13T14:15:29.567714+0000","last_peered":"2024-09-13T14:15:29.567714+0000","last_clean":"2024-09-13T14:15:29.567714+0000","last_became_active":"2024-09-13T14:15:26.569776+0000","last_became_peered":"2024-09-13T14:15:26.569776+0000","last_unstale":"2024-09-13T14:15:29.567714+0000","last_undegraded":"2024-09-13T14:15:29.567714+0000","last_fullsized":"2024-09-13T14:15:29.567714+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044993+0000","last_change":"2024-09-13T14:15:26.564796+0000","last_active":"2024-09-13T14:15:46.044993+0000","last_peered":"2024-09-13T14:15:46.044993+0000","last_clean":"2024-09-13T14:15:46.044993+0000","last_became_active":"2024-09-13T14:15:26.563959+0000","last_became_peered":"2024-09-13T14:15:26.563959+0000","last_unstale":"2024-09-13T14:15:46.044993+0000","last_undegraded":"2024-09-13T14:15:46.044993+0000","last_fullsized":"2024-09-13T14:15:46.044993+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044871+0000","last_change":"2024-09-13T14:15:26.567040+0000","last_active":"2024-09-13T14:15:46.044871+0000","last_peered":"2024-09-13T14:15:46.044871+0000","last_clean":"2024-09-13T14:15:46.044871+0000","last_became_active":"2024-09-13T14:15:26.566726+0000","last_became_peered":"2024-09-13T14:15:26.566726+0000","last_unstale":"2024-09-13T14:15:46.044871+0000","last_undegraded":"2024-09-13T14:15:46.044871+0000","last_fullsized":"2024-09-13T14:15:46.044871+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598208+0000","last_change":"2024-09-13T14:15:26.563679+0000","last_active":"2024-09-13T14:15:45.598208+0000","last_peered":"2024-09-13T14:15:45.598208+0000","last_clean":"2024-09-13T14:15:45.598208+0000","last_became_active":"2024-09-13T14:15:26.563402+0000","last_became_peered":"2024-09-13T14:15:26.563402+0000","last_unstale":"2024-09-13T14:15:45.598208+0000","last_undegraded":"2024-09-13T14:15:45.598208+0000","last_fullsized":"2024-09-13T14:15:45.598208+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597444+0000","last_change":"2024-09-13T14:15:26.552634+0000","last_active":"2024-09-13T14:15:45.597444+0000","last_peered":"2024-09-13T14:15:45.597444+0000","last_clean":"2024-09-13T14:15:45.597444+0000","last_became_active":"2024-09-13T14:15:26.552354+0000","last_became_peered":"2024-09-13T14:15:26.552354+0000","last_unstale":"2024-09-13T14:15:45.597444+0000","last_undegraded":"2024-09-13T14:15:45.597444+0000","last_fullsized":"2024-09-13T14:15:45.597444+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565423+0000","last_change":"2024-09-13T14:15:26.576136+0000","last_active":"2024-09-13T14:15:29.565423+0000","last_peered":"2024-09-13T14:15:29.565423+0000","last_clean":"2024-09-13T14:15:29.565423+0000","last_became_active":"2024-09-13T14:15:26.575737+0000","last_became_peered":"2024-09-13T14:15:26.575737+0000","last_unstale":"2024-09-13T14:15:29.565423+0000","last_undegraded":"2024-09-13T14:15:29.565423+0000","last_fullsized":"2024-09-13T14:15:29.565423+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566202+0000","last_change":"2024-09-13T14:15:26.566237+0000","last_active":"2024-09-13T14:15:29.566202+0000","last_peered":"2024-09-13T14:15:29.566202+0000","last_clean":"2024-09-13T14:15:29.566202+0000","last_became_active":"2024-09-13T14:15:26.565645+0000","last_became_peered":"2024-09-13T14:15:26.565645+0000","last_unstale":"2024-09-13T14:15:29.566202+0000","last_undegraded":"2024-09-13T14:15:29.566202+0000","last_fullsized":"2024-09-13T14:15:29.566202+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.965107+0000","last_change":"2024-09-13T14:15:46.965107+0000","last_active":"2024-09-13T14:15:46.965107+0000","last_peered":"2024-09-13T14:15:46.965107+0000","last_clean":"2024-09-13T14:15:46.965107+0000","last_became_active":"2024-09-13T14:15:46.964708+0000","last_became_peered":"2024-09-13T14:15:46.964708+0000","last_unstale":"2024-09-13T14:15:46.965107+0000","last_undegraded":"2024-09-13T14:15:46.965107+0000","last_fullsized":"2024-09-13T14:15:46.965107+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601904+0000","last_change":"2024-09-13T14:15:26.565535+0000","last_active":"2024-09-13T14:15:45.601904+0000","last_peered":"2024-09-13T14:15:45.601904+0000","last_clean":"2024-09-13T14:15:45.601904+0000","last_became_active":"2024-09-13T14:15:26.565091+0000","last_became_peered":"2024-09-13T14:15:26.565091+0000","last_unstale":"2024-09-13T14:15:45.601904+0000","last_undegraded":"2024-09-13T14:15:45.601904+0000","last_fullsized":"2024-09-13T14:15:45.601904+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.569840+0000","last_change":"2024-09-13T14:15:26.556686+0000","last_active":"2024-09-13T14:15:29.569840+0000","last_peered":"2024-09-13T14:15:29.569840+0000","last_clean":"2024-09-13T14:15:29.569840+0000","last_became_active":"2024-09-13T14:15:26.556349+0000","last_became_peered":"2024-09-13T14:15:26.556349+0000","last_unstale":"2024-09-13T14:15:29.569840+0000","last_undegraded":"2024-09-13T14:15:29.569840+0000","last_fullsized":"2024-09-13T14:15:29.569840+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567807+0000","last_change":"2024-09-13T14:15:26.564825+0000","last_active":"2024-09-13T14:15:29.567807+0000","last_peered":"2024-09-13T14:15:29.567807+0000","last_clean":"2024-09-13T14:15:29.567807+0000","last_became_active":"2024-09-13T14:15:26.563703+0000","last_became_peered":"2024-09-13T14:15:26.563703+0000","last_unstale":"2024-09-13T14:15:29.567807+0000","last_undegraded":"2024-09-13T14:15:29.567807+0000","last_fullsized":"2024-09-13T14:15:29.567807+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044809+0000","last_change":"2024-09-13T14:15:26.568327+0000","last_active":"2024-09-13T14:15:46.044809+0000","last_peered":"2024-09-13T14:15:46.044809+0000","last_clean":"2024-09-13T14:15:46.044809+0000","last_became_active":"2024-09-13T14:15:26.568070+0000","last_became_peered":"2024-09-13T14:15:26.568070+0000","last_unstale":"2024-09-13T14:15:46.044809+0000","last_undegraded":"2024-09-13T14:15:46.044809+0000","last_fullsized":"2024-09-13T14:15:46.044809+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565693+0000","last_change":"2024-09-13T14:15:26.568339+0000","last_active":"2024-09-13T14:15:29.565693+0000","last_peered":"2024-09-13T14:15:29.565693+0000","last_clean":"2024-09-13T14:15:29.565693+0000","last_became_active":"2024-09-13T14:15:26.568056+0000","last_became_peered":"2024-09-13T14:15:26.568056+0000","last_unstale":"2024-09-13T14:15:29.565693+0000","last_undegraded":"2024-09-13T14:15:29.565693+0000","last_fullsized":"2024-09-13T14:15:29.565693+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601444+0000","last_change":"2024-09-13T14:15:26.572417+0000","last_active":"2024-09-13T14:15:45.601444+0000","last_peered":"2024-09-13T14:15:45.601444+0000","last_clean":"2024-09-13T14:15:45.601444+0000","last_became_active":"2024-09-13T14:15:26.572126+0000","last_became_peered":"2024-09-13T14:15:26.572126+0000","last_unstale":"2024-09-13T14:15:45.601444+0000","last_undegraded":"2024-09-13T14:15:45.601444+0000","last_fullsized":"2024-09-13T14:15:45.601444+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597141+0000","last_change":"2024-09-13T14:15:26.555819+0000","last_active":"2024-09-13T14:15:45.597141+0000","last_peered":"2024-09-13T14:15:45.597141+0000","last_clean":"2024-09-13T14:15:45.597141+0000","last_became_active":"2024-09-13T14:15:26.555172+0000","last_became_peered":"2024-09-13T14:15:26.555172+0000","last_unstale":"2024-09-13T14:15:45.597141+0000","last_undegraded":"2024-09-13T14:15:45.597141+0000","last_fullsized":"2024-09-13T14:15:45.597141+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601257+0000","last_change":"2024-09-13T14:15:26.561070+0000","last_active":"2024-09-13T14:15:45.601257+0000","last_peered":"2024-09-13T14:15:45.601257+0000","last_clean":"2024-09-13T14:15:45.601257+0000","last_became_active":"2024-09-13T14:15:26.559771+0000","last_became_peered":"2024-09-13T14:15:26.559771+0000","last_unstale":"2024-09-13T14:15:45.601257+0000","last_undegraded":"2024-09-13T14:15:45.601257+0000","last_fullsized":"2024-09-13T14:15:45.601257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597763+0000","last_change":"2024-09-13T14:15:26.554841+0000","last_active":"2024-09-13T14:15:45.597763+0000","last_peered":"2024-09-13T14:15:45.597763+0000","last_clean":"2024-09-13T14:15:45.597763+0000","last_became_active":"2024-09-13T14:15:26.553231+0000","last_became_peered":"2024-09-13T14:15:26.553231+0000","last_unstale":"2024-09-13T14:15:45.597763+0000","last_undegraded":"2024-09-13T14:15:45.597763+0000","last_fullsized":"2024-09-13T14:15:45.597763+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045061+0000","last_change":"2024-09-13T14:15:26.551060+0000","last_active":"2024-09-13T14:15:46.045061+0000","last_peered":"2024-09-13T14:15:46.045061+0000","last_clean":"2024-09-13T14:15:46.045061+0000","last_became_active":"2024-09-13T14:15:26.550739+0000","last_became_peered":"2024-09-13T14:15:26.550739+0000","last_unstale":"2024-09-13T14:15:46.045061+0000","last_undegraded":"2024-09-13T14:15:46.045061+0000","last_fullsized":"2024-09-13T14:15:46.045061+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566588+0000","last_change":"2024-09-13T14:15:26.559670+0000","last_active":"2024-09-13T14:15:29.566588+0000","last_peered":"2024-09-13T14:15:29.566588+0000","last_clean":"2024-09-13T14:15:29.566588+0000","last_became_active":"2024-09-13T14:15:26.559401+0000","last_became_peered":"2024-09-13T14:15:26.559401+0000","last_unstale":"2024-09-13T14:15:29.566588+0000","last_undegraded":"2024-09-13T14:15:29.566588+0000","last_fullsized":"2024-09-13T14:15:29.566588+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597525+0000","last_change":"2024-09-13T14:15:26.570832+0000","last_active":"2024-09-13T14:15:45.597525+0000","last_peered":"2024-09-13T14:15:45.597525+0000","last_clean":"2024-09-13T14:15:45.597525+0000","last_became_active":"2024-09-13T14:15:26.570531+0000","last_became_peered":"2024-09-13T14:15:26.570531+0000","last_unstale":"2024-09-13T14:15:45.597525+0000","last_undegraded":"2024-09-13T14:15:45.597525+0000","last_fullsized":"2024-09-13T14:15:45.597525+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565724+0000","last_change":"2024-09-13T14:15:26.573053+0000","last_active":"2024-09-13T14:15:29.565724+0000","last_peered":"2024-09-13T14:15:29.565724+0000","last_clean":"2024-09-13T14:15:29.565724+0000","last_became_active":"2024-09-13T14:15:26.572783+0000","last_became_peered":"2024-09-13T14:15:26.572783+0000","last_unstale":"2024-09-13T14:15:29.565724+0000","last_undegraded":"2024-09-13T14:15:29.565724+0000","last_fullsized":"2024-09-13T14:15:29.565724+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601363+0000","last_change":"2024-09-13T14:15:26.569346+0000","last_active":"2024-09-13T14:15:45.601363+0000","last_peered":"2024-09-13T14:15:45.601363+0000","last_clean":"2024-09-13T14:15:45.601363+0000","last_became_active":"2024-09-13T14:15:26.568939+0000","last_became_peered":"2024-09-13T14:15:26.568939+0000","last_unstale":"2024-09-13T14:15:45.601363+0000","last_undegraded":"2024-09-13T14:15:45.601363+0000","last_fullsized":"2024-09-13T14:15:45.601363+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565950+0000","last_change":"2024-09-13T14:15:26.562543+0000","last_active":"2024-09-13T14:15:29.565950+0000","last_peered":"2024-09-13T14:15:29.565950+0000","last_clean":"2024-09-13T14:15:29.565950+0000","last_became_active":"2024-09-13T14:15:26.562267+0000","last_became_peered":"2024-09-13T14:15:26.562267+0000","last_unstale":"2024-09-13T14:15:29.565950+0000","last_undegraded":"2024-09-13T14:15:29.565950+0000","last_fullsized":"2024-09-13T14:15:29.565950+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044457+0000","last_change":"2024-09-13T14:15:26.568440+0000","last_active":"2024-09-13T14:15:46.044457+0000","last_peered":"2024-09-13T14:15:46.044457+0000","last_clean":"2024-09-13T14:15:46.044457+0000","last_became_active":"2024-09-13T14:15:26.568169+0000","last_became_peered":"2024-09-13T14:15:26.568169+0000","last_unstale":"2024-09-13T14:15:46.044457+0000","last_undegraded":"2024-09-13T14:15:46.044457+0000","last_fullsized":"2024-09-13T14:15:46.044457+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":4,"reported_epoch":59,"state":"activating","last_fresh":"2024-09-13T14:15:46.599009+0000","last_change":"2024-09-13T14:15:46.599009+0000","last_active":"2024-09-13T14:15:25.534586+0000","last_peered":"2024-09-13T14:15:25.534586+0000","last_clean":"2024-09-13T14:15:25.534586+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-13T14:15:46.599009+0000","last_undegraded":"2024-09-13T14:15:46.599009+0000","last_fullsized":"2024-09-13T14:15:46.599009+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":["3","1","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597677+0000","last_change":"2024-09-13T14:15:22.541205+0000","last_active":"2024-09-13T14:15:45.597677+0000","last_peered":"2024-09-13T14:15:45.597677+0000","last_clean":"2024-09-13T14:15:45.597677+0000","last_became_active":"2024-09-13T14:15:22.539574+0000","last_became_peered":"2024-09-13T14:15:22.539574+0000","last_unstale":"2024-09-13T14:15:45.597677+0000","last_undegraded":"2024-09-13T14:15:45.597677+0000","last_fullsized":"2024-09-13T14:15:45.597677+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-13T14:15:45.599425+0000","last_change":"2024-09-13T14:15:45.599425+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-13T14:15:45.599425+0000","last_undegraded":"2024-09-13T14:15:45.599425+0000","last_fullsized":"2024-09-13T14:15:45.599425+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:43.338321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044687+0000","last_change":"2024-09-13T14:15:22.543444+0000","last_active":"2024-09-13T14:15:46.044687+0000","last_peered":"2024-09-13T14:15:46.044687+0000","last_clean":"2024-09-13T14:15:46.044687+0000","last_became_active":"2024-09-13T14:15:22.543138+0000","last_became_peered":"2024-09-13T14:15:22.543138+0000","last_unstale":"2024-09-13T14:15:46.044687+0000","last_undegraded":"2024-09-13T14:15:46.044687+0000","last_fullsized":"2024-09-13T14:15:46.044687+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597273+0000","last_change":"2024-09-13T14:15:22.541457+0000","last_active":"2024-09-13T14:15:45.597273+0000","last_peered":"2024-09-13T14:15:45.597273+0000","last_clean":"2024-09-13T14:15:45.597273+0000","last_became_active":"2024-09-13T14:15:22.539753+0000","last_became_peered":"2024-09-13T14:15:22.539753+0000","last_unstale":"2024-09-13T14:15:45.597273+0000","last_undegraded":"2024-09-13T14:15:45.597273+0000","last_fullsized":"2024-09-13T14:15:45.597273+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596847+0000","last_change":"2024-09-13T14:15:26.571137+0000","last_active":"2024-09-13T14:15:45.596847+0000","last_peered":"2024-09-13T14:15:45.596847+0000","last_clean":"2024-09-13T14:15:45.596847+0000","last_became_active":"2024-09-13T14:15:26.570884+0000","last_became_peered":"2024-09-13T14:15:26.570884+0000","last_unstale":"2024-09-13T14:15:45.596847+0000","last_undegraded":"2024-09-13T14:15:45.596847+0000","last_fullsized":"2024-09-13T14:15:45.596847+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601578+0000","last_change":"2024-09-13T14:15:22.546875+0000","last_active":"2024-09-13T14:15:45.601578+0000","last_peered":"2024-09-13T14:15:45.601578+0000","last_clean":"2024-09-13T14:15:45.601578+0000","last_became_active":"2024-09-13T14:15:22.545863+0000","last_became_peered":"2024-09-13T14:15:22.545863+0000","last_unstale":"2024-09-13T14:15:45.601578+0000","last_undegraded":"2024-09-13T14:15:45.601578+0000","last_fullsized":"2024-09-13T14:15:45.601578+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565078+0000","last_change":"2024-09-13T14:15:26.560575+0000","last_active":"2024-09-13T14:15:29.565078+0000","last_peered":"2024-09-13T14:15:29.565078+0000","last_clean":"2024-09-13T14:15:29.565078+0000","last_became_active":"2024-09-13T14:15:26.560291+0000","last_became_peered":"2024-09-13T14:15:26.560291+0000","last_unstale":"2024-09-13T14:15:29.565078+0000","last_undegraded":"2024-09-13T14:15:29.565078+0000","last_fullsized":"2024-09-13T14:15:29.565078+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045029+0000","last_change":"2024-09-13T14:15:22.546137+0000","last_active":"2024-09-13T14:15:46.045029+0000","last_peered":"2024-09-13T14:15:46.045029+0000","last_clean":"2024-09-13T14:15:46.045029+0000","last_became_active":"2024-09-13T14:15:22.545833+0000","last_became_peered":"2024-09-13T14:15:22.545833+0000","last_unstale":"2024-09-13T14:15:46.045029+0000","last_undegraded":"2024-09-13T14:15:46.045029+0000","last_fullsized":"2024-09-13T14:15:46.045029+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602112+0000","last_change":"2024-09-13T14:15:26.566451+0000","last_active":"2024-09-13T14:15:45.602112+0000","last_peered":"2024-09-13T14:15:45.602112+0000","last_clean":"2024-09-13T14:15:45.602112+0000","last_became_active":"2024-09-13T14:15:26.565363+0000","last_became_peered":"2024-09-13T14:15:26.565363+0000","last_unstale":"2024-09-13T14:15:45.602112+0000","last_undegraded":"2024-09-13T14:15:45.602112+0000","last_fullsized":"2024-09-13T14:15:45.602112+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602192+0000","last_change":"2024-09-13T14:15:22.540966+0000","last_active":"2024-09-13T14:15:45.602192+0000","last_peered":"2024-09-13T14:15:45.602192+0000","last_clean":"2024-09-13T14:15:45.602192+0000","last_became_active":"2024-09-13T14:15:22.540611+0000","last_became_peered":"2024-09-13T14:15:22.540611+0000","last_unstale":"2024-09-13T14:15:45.602192+0000","last_undegraded":"2024-09-13T14:15:45.602192+0000","last_fullsized":"2024-09-13T14:15:45.602192+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565706+0000","last_change":"2024-09-13T14:15:26.573426+0000","last_active":"2024-09-13T14:15:29.565706+0000","last_peered":"2024-09-13T14:15:29.565706+0000","last_clean":"2024-09-13T14:15:29.565706+0000","last_became_active":"2024-09-13T14:15:26.573138+0000","last_became_peered":"2024-09-13T14:15:26.573138+0000","last_unstale":"2024-09-13T14:15:29.565706+0000","last_undegraded":"2024-09-13T14:15:29.565706+0000","last_fullsized":"2024-09-13T14:15:29.565706+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":18,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.571949+0000","last_change":"2024-09-13T14:15:22.543031+0000","last_active":"2024-09-13T14:15:29.571949+0000","last_peered":"2024-09-13T14:15:29.571949+0000","last_clean":"2024-09-13T14:15:29.571949+0000","last_became_active":"2024-09-13T14:15:22.542739+0000","last_became_peered":"2024-09-13T14:15:22.542739+0000","last_unstale":"2024-09-13T14:15:29.571949+0000","last_undegraded":"2024-09-13T14:15:29.571949+0000","last_fullsized":"2024-09-13T14:15:29.571949+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598301+0000","last_change":"2024-09-13T14:15:26.551575+0000","last_active":"2024-09-13T14:15:45.598301+0000","last_peered":"2024-09-13T14:15:45.598301+0000","last_clean":"2024-09-13T14:15:45.598301+0000","last_became_active":"2024-09-13T14:15:26.550913+0000","last_became_peered":"2024-09-13T14:15:26.550913+0000","last_unstale":"2024-09-13T14:15:45.598301+0000","last_undegraded":"2024-09-13T14:15:45.598301+0000","last_fullsized":"2024-09-13T14:15:45.598301+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044871+0000","last_change":"2024-09-13T14:15:22.539191+0000","last_active":"2024-09-13T14:15:46.044871+0000","last_peered":"2024-09-13T14:15:46.044871+0000","last_clean":"2024-09-13T14:15:46.044871+0000","last_became_active":"2024-09-13T14:15:22.538864+0000","last_became_peered":"2024-09-13T14:15:22.538864+0000","last_unstale":"2024-09-13T14:15:46.044871+0000","last_undegraded":"2024-09-13T14:15:46.044871+0000","last_fullsized":"2024-09-13T14:15:46.044871+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567611+0000","last_change":"2024-09-13T14:15:26.567770+0000","last_active":"2024-09-13T14:15:29.567611+0000","last_peered":"2024-09-13T14:15:29.567611+0000","last_clean":"2024-09-13T14:15:29.567611+0000","last_became_active":"2024-09-13T14:15:26.567513+0000","last_became_peered":"2024-09-13T14:15:26.567513+0000","last_unstale":"2024-09-13T14:15:29.567611+0000","last_undegraded":"2024-09-13T14:15:29.567611+0000","last_fullsized":"2024-09-13T14:15:29.567611+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.572467+0000","last_change":"2024-09-13T14:15:22.545520+0000","last_active":"2024-09-13T14:15:29.572467+0000","last_peered":"2024-09-13T14:15:29.572467+0000","last_clean":"2024-09-13T14:15:29.572467+0000","last_became_active":"2024-09-13T14:15:22.545135+0000","last_became_peered":"2024-09-13T14:15:22.545135+0000","last_unstale":"2024-09-13T14:15:29.572467+0000","last_undegraded":"2024-09-13T14:15:29.572467+0000","last_fullsized":"2024-09-13T14:15:29.572467+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565531+0000","last_change":"2024-09-13T14:15:26.575023+0000","last_active":"2024-09-13T14:15:29.565531+0000","last_peered":"2024-09-13T14:15:29.565531+0000","last_clean":"2024-09-13T14:15:29.565531+0000","last_became_active":"2024-09-13T14:15:26.574648+0000","last_became_peered":"2024-09-13T14:15:26.574648+0000","last_unstale":"2024-09-13T14:15:29.565531+0000","last_undegraded":"2024-09-13T14:15:29.565531+0000","last_fullsized":"2024-09-13T14:15:29.565531+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601454+0000","last_change":"2024-09-13T14:15:22.546988+0000","last_active":"2024-09-13T14:15:45.601454+0000","last_peered":"2024-09-13T14:15:45.601454+0000","last_clean":"2024-09-13T14:15:45.601454+0000","last_became_active":"2024-09-13T14:15:22.545255+0000","last_became_peered":"2024-09-13T14:15:22.545255+0000","last_unstale":"2024-09-13T14:15:45.601454+0000","last_undegraded":"2024-09-13T14:15:45.601454+0000","last_fullsized":"2024-09-13T14:15:45.601454+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567518+0000","last_change":"2024-09-13T14:15:26.555682+0000","last_active":"2024-09-13T14:15:29.567518+0000","last_peered":"2024-09-13T14:15:29.567518+0000","last_clean":"2024-09-13T14:15:29.567518+0000","last_became_active":"2024-09-13T14:15:26.555391+0000","last_became_peered":"2024-09-13T14:15:26.555391+0000","last_unstale":"2024-09-13T14:15:29.567518+0000","last_undegraded":"2024-09-13T14:15:29.567518+0000","last_fullsized":"2024-09-13T14:15:29.567518+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044543+0000","last_change":"2024-09-13T14:15:22.542050+0000","last_active":"2024-09-13T14:15:46.044543+0000","last_peered":"2024-09-13T14:15:46.044543+0000","last_clean":"2024-09-13T14:15:46.044543+0000","last_became_active":"2024-09-13T14:15:22.541726+0000","last_became_peered":"2024-09-13T14:15:22.541726+0000","last_unstale":"2024-09-13T14:15:46.044543+0000","last_undegraded":"2024-09-13T14:15:46.044543+0000","last_fullsized":"2024-09-13T14:15:46.044543+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044523+0000","last_change":"2024-09-13T14:15:26.567258+0000","last_active":"2024-09-13T14:15:46.044523+0000","last_peered":"2024-09-13T14:15:46.044523+0000","last_clean":"2024-09-13T14:15:46.044523+0000","last_became_active":"2024-09-13T14:15:26.566924+0000","last_became_peered":"2024-09-13T14:15:26.566924+0000","last_unstale":"2024-09-13T14:15:46.044523+0000","last_undegraded":"2024-09-13T14:15:46.044523+0000","last_fullsized":"2024-09-13T14:15:46.044523+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":23,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.573457+0000","last_change":"2024-09-13T14:15:22.541058+0000","last_active":"2024-09-13T14:15:29.573457+0000","last_peered":"2024-09-13T14:15:29.573457+0000","last_clean":"2024-09-13T14:15:29.573457+0000","last_became_active":"2024-09-13T14:15:22.540793+0000","last_became_peered":"2024-09-13T14:15:22.540793+0000","last_unstale":"2024-09-13T14:15:29.573457+0000","last_undegraded":"2024-09-13T14:15:29.573457+0000","last_fullsized":"2024-09-13T14:15:29.573457+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565361+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:15:29.565361+0000","last_peered":"2024-09-13T14:15:29.565361+0000","last_clean":"2024-09-13T14:15:29.565361+0000","last_became_active":"2024-09-13T14:15:26.562669+0000","last_became_peered":"2024-09-13T14:15:26.562669+0000","last_unstale":"2024-09-13T14:15:29.565361+0000","last_undegraded":"2024-09-13T14:15:29.565361+0000","last_fullsized":"2024-09-13T14:15:29.565361+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598460+0000","last_change":"2024-09-13T14:15:22.540681+0000","last_active":"2024-09-13T14:15:45.598460+0000","last_peered":"2024-09-13T14:15:45.598460+0000","last_clean":"2024-09-13T14:15:45.598460+0000","last_became_active":"2024-09-13T14:15:22.540386+0000","last_became_peered":"2024-09-13T14:15:22.540386+0000","last_unstale":"2024-09-13T14:15:45.598460+0000","last_undegraded":"2024-09-13T14:15:45.598460+0000","last_fullsized":"2024-09-13T14:15:45.598460+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567376+0000","last_change":"2024-09-13T14:15:26.559281+0000","last_active":"2024-09-13T14:15:29.567376+0000","last_peered":"2024-09-13T14:15:29.567376+0000","last_clean":"2024-09-13T14:15:29.567376+0000","last_became_active":"2024-09-13T14:15:26.558987+0000","last_became_peered":"2024-09-13T14:15:26.558987+0000","last_unstale":"2024-09-13T14:15:29.567376+0000","last_undegraded":"2024-09-13T14:15:29.567376+0000","last_fullsized":"2024-09-13T14:15:29.567376+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.573411+0000","last_change":"2024-09-13T14:15:22.544147+0000","last_active":"2024-09-13T14:15:29.573411+0000","last_peered":"2024-09-13T14:15:29.573411+0000","last_clean":"2024-09-13T14:15:29.573411+0000","last_became_active":"2024-09-13T14:15:22.543782+0000","last_became_peered":"2024-09-13T14:15:22.543782+0000","last_unstale":"2024-09-13T14:15:29.573411+0000","last_undegraded":"2024-09-13T14:15:29.573411+0000","last_fullsized":"2024-09-13T14:15:29.573411+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566113+0000","last_change":"2024-09-13T14:15:26.570603+0000","last_active":"2024-09-13T14:15:29.566113+0000","last_peered":"2024-09-13T14:15:29.566113+0000","last_clean":"2024-09-13T14:15:29.566113+0000","last_became_active":"2024-09-13T14:15:26.569357+0000","last_became_peered":"2024-09-13T14:15:26.569357+0000","last_unstale":"2024-09-13T14:15:29.566113+0000","last_undegraded":"2024-09-13T14:15:29.566113+0000","last_fullsized":"2024-09-13T14:15:29.566113+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.571838+0000","last_change":"2024-09-13T14:15:22.546363+0000","last_active":"2024-09-13T14:15:29.571838+0000","last_peered":"2024-09-13T14:15:29.571838+0000","last_clean":"2024-09-13T14:15:29.571838+0000","last_became_active":"2024-09-13T14:15:22.544417+0000","last_became_peered":"2024-09-13T14:15:22.544417+0000","last_unstale":"2024-09-13T14:15:29.571838+0000","last_undegraded":"2024-09-13T14:15:29.571838+0000","last_fullsized":"2024-09-13T14:15:29.571838+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601813+0000","last_change":"2024-09-13T14:15:26.572445+0000","last_active":"2024-09-13T14:15:45.601813+0000","last_peered":"2024-09-13T14:15:45.601813+0000","last_clean":"2024-09-13T14:15:45.601813+0000","last_became_active":"2024-09-13T14:15:26.572166+0000","last_became_peered":"2024-09-13T14:15:26.572166+0000","last_unstale":"2024-09-13T14:15:45.601813+0000","last_undegraded":"2024-09-13T14:15:45.601813+0000","last_fullsized":"2024-09-13T14:15:45.601813+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.572250+0000","last_change":"2024-09-13T14:15:22.542920+0000","last_active":"2024-09-13T14:15:29.572250+0000","last_peered":"2024-09-13T14:15:29.572250+0000","last_clean":"2024-09-13T14:15:29.572250+0000","last_became_active":"2024-09-13T14:15:22.542641+0000","last_became_peered":"2024-09-13T14:15:22.542641+0000","last_unstale":"2024-09-13T14:15:29.572250+0000","last_undegraded":"2024-09-13T14:15:29.572250+0000","last_fullsized":"2024-09-13T14:15:29.572250+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567695+0000","last_change":"2024-09-13T14:15:26.572195+0000","last_active":"2024-09-13T14:15:29.567695+0000","last_peered":"2024-09-13T14:15:29.567695+0000","last_clean":"2024-09-13T14:15:29.567695+0000","last_became_active":"2024-09-13T14:15:26.571003+0000","last_became_peered":"2024-09-13T14:15:26.571003+0000","last_unstale":"2024-09-13T14:15:29.567695+0000","last_undegraded":"2024-09-13T14:15:29.567695+0000","last_fullsized":"2024-09-13T14:15:29.567695+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":24,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565327+0000","last_change":"2024-09-13T14:14:57.479980+0000","last_active":"2024-09-13T14:15:29.565327+0000","last_peered":"2024-09-13T14:15:29.565327+0000","last_clean":"2024-09-13T14:15:29.565327+0000","last_became_active":"2024-09-13T14:14:57.475298+0000","last_became_peered":"2024-09-13T14:14:57.475298+0000","last_unstale":"2024-09-13T14:15:29.565327+0000","last_undegraded":"2024-09-13T14:15:29.565327+0000","last_fullsized":"2024-09-13T14:15:29.565327+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_clean_scrub_stamp":"2024-09-13T14:12:56.059430+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601202+0000","last_change":"2024-09-13T14:15:22.542042+0000","last_active":"2024-09-13T14:15:45.601202+0000","last_peered":"2024-09-13T14:15:45.601202+0000","last_clean":"2024-09-13T14:15:45.601202+0000","last_became_active":"2024-09-13T14:15:22.541756+0000","last_became_peered":"2024-09-13T14:15:22.541756+0000","last_unstale":"2024-09-13T14:15:45.601202+0000","last_undegraded":"2024-09-13T14:15:45.601202+0000","last_fullsized":"2024-09-13T14:15:45.601202+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597402+0000","last_change":"2024-09-13T14:15:26.565510+0000","last_active":"2024-09-13T14:15:45.597402+0000","last_peered":"2024-09-13T14:15:45.597402+0000","last_clean":"2024-09-13T14:15:45.597402+0000","last_became_active":"2024-09-13T14:15:26.565213+0000","last_became_peered":"2024-09-13T14:15:26.565213+0000","last_unstale":"2024-09-13T14:15:45.597402+0000","last_undegraded":"2024-09-13T14:15:45.597402+0000","last_fullsized":"2024-09-13T14:15:45.597402+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565280+0000","last_change":"2024-09-13T14:15:22.550379+0000","last_active":"2024-09-13T14:15:29.565280+0000","last_peered":"2024-09-13T14:15:29.565280+0000","last_clean":"2024-09-13T14:15:29.565280+0000","last_became_active":"2024-09-13T14:15:22.549995+0000","last_became_peered":"2024-09-13T14:15:22.549995+0000","last_unstale":"2024-09-13T14:15:29.565280+0000","last_undegraded":"2024-09-13T14:15:29.565280+0000","last_fullsized":"2024-09-13T14:15:29.565280+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-13T14:15:45.599837+0000","last_change":"2024-09-13T14:15:45.600389+0000","last_active":"2024-09-13T14:15:45.599806+0000","last_peered":"2024-09-13T14:15:29.564907+0000","last_clean":"2024-09-13T14:15:29.564907+0000","last_became_active":"2024-09-13T14:15:26.573416+0000","last_became_peered":"2024-09-13T14:15:26.573416+0000","last_unstale":"2024-09-13T14:15:45.599837+0000","last_undegraded":"2024-09-13T14:15:45.599837+0000","last_fullsized":"2024-09-13T14:15:45.599837+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:10.597103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,4],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.569048+0000","last_change":"2024-09-13T14:15:22.544154+0000","last_active":"2024-09-13T14:15:29.569048+0000","last_peered":"2024-09-13T14:15:29.569048+0000","last_clean":"2024-09-13T14:15:29.569048+0000","last_became_active":"2024-09-13T14:15:22.543880+0000","last_became_peered":"2024-09-13T14:15:22.543880+0000","last_unstale":"2024-09-13T14:15:29.569048+0000","last_undegraded":"2024-09-13T14:15:29.569048+0000","last_fullsized":"2024-09-13T14:15:29.569048+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565078+0000","last_change":"2024-09-13T14:15:26.553134+0000","last_active":"2024-09-13T14:15:29.565078+0000","last_peered":"2024-09-13T14:15:29.565078+0000","last_clean":"2024-09-13T14:15:29.565078+0000","last_became_active":"2024-09-13T14:15:26.552054+0000","last_became_peered":"2024-09-13T14:15:26.552054+0000","last_unstale":"2024-09-13T14:15:29.565078+0000","last_undegraded":"2024-09-13T14:15:29.565078+0000","last_fullsized":"2024-09-13T14:15:29.565078+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601553+0000","last_change":"2024-09-13T14:15:22.545901+0000","last_active":"2024-09-13T14:15:45.601553+0000","last_peered":"2024-09-13T14:15:45.601553+0000","last_clean":"2024-09-13T14:15:45.601553+0000","last_became_active":"2024-09-13T14:15:22.545610+0000","last_became_peered":"2024-09-13T14:15:22.545610+0000","last_unstale":"2024-09-13T14:15:45.601553+0000","last_undegraded":"2024-09-13T14:15:45.601553+0000","last_fullsized":"2024-09-13T14:15:45.601553+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598680+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:15:45.598680+0000","last_peered":"2024-09-13T14:15:45.598680+0000","last_clean":"2024-09-13T14:15:45.598680+0000","last_became_active":"2024-09-13T14:15:26.562679+0000","last_became_peered":"2024-09-13T14:15:26.562679+0000","last_unstale":"2024-09-13T14:15:45.598680+0000","last_undegraded":"2024-09-13T14:15:45.598680+0000","last_fullsized":"2024-09-13T14:15:45.598680+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566706+0000","last_change":"2024-09-13T14:15:22.549591+0000","last_active":"2024-09-13T14:15:29.566706+0000","last_peered":"2024-09-13T14:15:29.566706+0000","last_clean":"2024-09-13T14:15:29.566706+0000","last_became_active":"2024-09-13T14:15:22.549305+0000","last_became_peered":"2024-09-13T14:15:22.549305+0000","last_unstale":"2024-09-13T14:15:29.566706+0000","last_undegraded":"2024-09-13T14:15:29.566706+0000","last_fullsized":"2024-09-13T14:15:29.566706+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601943+0000","last_change":"2024-09-13T14:15:26.562976+0000","last_active":"2024-09-13T14:15:45.601943+0000","last_peered":"2024-09-13T14:15:45.601943+0000","last_clean":"2024-09-13T14:15:45.601943+0000","last_became_active":"2024-09-13T14:15:26.561391+0000","last_became_peered":"2024-09-13T14:15:26.561391+0000","last_unstale":"2024-09-13T14:15:45.601943+0000","last_undegraded":"2024-09-13T14:15:45.601943+0000","last_fullsized":"2024-09-13T14:15:45.601943+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.567995+0000","last_change":"2024-09-13T14:15:22.550059+0000","last_active":"2024-09-13T14:15:29.567995+0000","last_peered":"2024-09-13T14:15:29.567995+0000","last_clean":"2024-09-13T14:15:29.567995+0000","last_became_active":"2024-09-13T14:15:22.549682+0000","last_became_peered":"2024-09-13T14:15:22.549682+0000","last_unstale":"2024-09-13T14:15:29.567995+0000","last_undegraded":"2024-09-13T14:15:29.567995+0000","last_fullsized":"2024-09-13T14:15:29.567995+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566616+0000","last_change":"2024-09-13T14:15:26.565253+0000","last_active":"2024-09-13T14:15:29.566616+0000","last_peered":"2024-09-13T14:15:29.566616+0000","last_clean":"2024-09-13T14:15:29.566616+0000","last_became_active":"2024-09-13T14:15:26.564972+0000","last_became_peered":"2024-09-13T14:15:26.564972+0000","last_unstale":"2024-09-13T14:15:29.566616+0000","last_undegraded":"2024-09-13T14:15:29.566616+0000","last_fullsized":"2024-09-13T14:15:29.566616+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:32.214795+0000","last_change":"2024-09-13T14:15:22.547223+0000","last_active":"2024-09-13T14:15:32.214795+0000","last_peered":"2024-09-13T14:15:32.214795+0000","last_clean":"2024-09-13T14:15:32.214795+0000","last_became_active":"2024-09-13T14:15:22.546934+0000","last_became_peered":"2024-09-13T14:15:22.546934+0000","last_unstale":"2024-09-13T14:15:32.214795+0000","last_undegraded":"2024-09-13T14:15:32.214795+0000","last_fullsized":"2024-09-13T14:15:32.214795+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597836+0000","last_change":"2024-09-13T14:15:26.571004+0000","last_active":"2024-09-13T14:15:45.597836+0000","last_peered":"2024-09-13T14:15:45.597836+0000","last_clean":"2024-09-13T14:15:45.597836+0000","last_became_active":"2024-09-13T14:15:26.570753+0000","last_became_peered":"2024-09-13T14:15:26.570753+0000","last_unstale":"2024-09-13T14:15:45.597836+0000","last_undegraded":"2024-09-13T14:15:45.597836+0000","last_fullsized":"2024-09-13T14:15:45.597836+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045057+0000","last_change":"2024-09-13T14:15:22.545342+0000","last_active":"2024-09-13T14:15:46.045057+0000","last_peered":"2024-09-13T14:15:46.045057+0000","last_clean":"2024-09-13T14:15:46.045057+0000","last_became_active":"2024-09-13T14:15:22.545067+0000","last_became_peered":"2024-09-13T14:15:22.545067+0000","last_unstale":"2024-09-13T14:15:46.045057+0000","last_undegraded":"2024-09-13T14:15:46.045057+0000","last_fullsized":"2024-09-13T14:15:46.045057+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-13T14:15:45.600133+0000","last_change":"2024-09-13T14:15:45.600133+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-13T14:15:45.600133+0000","last_undegraded":"2024-09-13T14:15:45.600133+0000","last_fullsized":"2024-09-13T14:15:45.600133+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:06:14.904239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565236+0000","last_change":"2024-09-13T14:15:22.550293+0000","last_active":"2024-09-13T14:15:29.565236+0000","last_peered":"2024-09-13T14:15:29.565236+0000","last_clean":"2024-09-13T14:15:29.565236+0000","last_became_active":"2024-09-13T14:15:22.549735+0000","last_became_peered":"2024-09-13T14:15:22.549735+0000","last_unstale":"2024-09-13T14:15:29.565236+0000","last_undegraded":"2024-09-13T14:15:29.565236+0000","last_fullsized":"2024-09-13T14:15:29.565236+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598085+0000","last_change":"2024-09-13T14:15:26.556457+0000","last_active":"2024-09-13T14:15:45.598085+0000","last_peered":"2024-09-13T14:15:45.598085+0000","last_clean":"2024-09-13T14:15:45.598085+0000","last_became_active":"2024-09-13T14:15:26.555684+0000","last_became_peered":"2024-09-13T14:15:26.555684+0000","last_unstale":"2024-09-13T14:15:45.598085+0000","last_undegraded":"2024-09-13T14:15:45.598085+0000","last_fullsized":"2024-09-13T14:15:45.598085+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566339+0000","last_change":"2024-09-13T14:15:22.549793+0000","last_active":"2024-09-13T14:15:29.566339+0000","last_peered":"2024-09-13T14:15:29.566339+0000","last_clean":"2024-09-13T14:15:29.566339+0000","last_became_active":"2024-09-13T14:15:22.549291+0000","last_became_peered":"2024-09-13T14:15:22.549291+0000","last_unstale":"2024-09-13T14:15:29.566339+0000","last_undegraded":"2024-09-13T14:15:29.566339+0000","last_fullsized":"2024-09-13T14:15:29.566339+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596926+0000","last_change":"2024-09-13T14:15:26.554955+0000","last_active":"2024-09-13T14:15:45.596926+0000","last_peered":"2024-09-13T14:15:45.596926+0000","last_clean":"2024-09-13T14:15:45.596926+0000","last_became_active":"2024-09-13T14:15:26.553125+0000","last_became_peered":"2024-09-13T14:15:26.553125+0000","last_unstale":"2024-09-13T14:15:45.596926+0000","last_undegraded":"2024-09-13T14:15:45.596926+0000","last_fullsized":"2024-09-13T14:15:45.596926+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.566559+0000","last_change":"2024-09-13T14:15:22.549798+0000","last_active":"2024-09-13T14:15:29.566559+0000","last_peered":"2024-09-13T14:15:29.566559+0000","last_clean":"2024-09-13T14:15:29.566559+0000","last_became_active":"2024-09-13T14:15:22.549508+0000","last_became_peered":"2024-09-13T14:15:22.549508+0000","last_unstale":"2024-09-13T14:15:29.566559+0000","last_undegraded":"2024-09-13T14:15:29.566559+0000","last_fullsized":"2024-09-13T14:15:29.566559+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597521+0000","last_change":"2024-09-13T14:15:26.571783+0000","last_active":"2024-09-13T14:15:45.597521+0000","last_peered":"2024-09-13T14:15:45.597521+0000","last_clean":"2024-09-13T14:15:45.597521+0000","last_became_active":"2024-09-13T14:15:26.571523+0000","last_became_peered":"2024-09-13T14:15:26.571523+0000","last_unstale":"2024-09-13T14:15:45.597521+0000","last_undegraded":"2024-09-13T14:15:45.597521+0000","last_fullsized":"2024-09-13T14:15:45.597521+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.570668+0000","last_change":"2024-09-13T14:15:22.549324+0000","last_active":"2024-09-13T14:15:29.570668+0000","last_peered":"2024-09-13T14:15:29.570668+0000","last_clean":"2024-09-13T14:15:29.570668+0000","last_became_active":"2024-09-13T14:15:22.549022+0000","last_became_peered":"2024-09-13T14:15:22.549022+0000","last_unstale":"2024-09-13T14:15:29.570668+0000","last_undegraded":"2024-09-13T14:15:29.570668+0000","last_fullsized":"2024-09-13T14:15:29.570668+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044615+0000","last_change":"2024-09-13T14:15:26.549648+0000","last_active":"2024-09-13T14:15:46.044615+0000","last_peered":"2024-09-13T14:15:46.044615+0000","last_clean":"2024-09-13T14:15:46.044615+0000","last_became_active":"2024-09-13T14:15:26.549328+0000","last_became_peered":"2024-09-13T14:15:26.549328+0000","last_unstale":"2024-09-13T14:15:46.044615+0000","last_undegraded":"2024-09-13T14:15:46.044615+0000","last_fullsized":"2024-09-13T14:15:46.044615+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601847+0000","last_change":"2024-09-13T14:15:26.573588+0000","last_active":"2024-09-13T14:15:45.601847+0000","last_peered":"2024-09-13T14:15:45.601847+0000","last_clean":"2024-09-13T14:15:45.601847+0000","last_became_active":"2024-09-13T14:15:26.572830+0000","last_became_peered":"2024-09-13T14:15:26.572830+0000","last_unstale":"2024-09-13T14:15:45.601847+0000","last_undegraded":"2024-09-13T14:15:45.601847+0000","last_fullsized":"2024-09-13T14:15:45.601847+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.572775+0000","last_change":"2024-09-13T14:15:22.550180+0000","last_active":"2024-09-13T14:15:29.572775+0000","last_peered":"2024-09-13T14:15:29.572775+0000","last_clean":"2024-09-13T14:15:29.572775+0000","last_became_active":"2024-09-13T14:15:22.549754+0000","last_became_peered":"2024-09-13T14:15:22.549754+0000","last_unstale":"2024-09-13T14:15:29.572775+0000","last_undegraded":"2024-09-13T14:15:29.572775+0000","last_fullsized":"2024-09-13T14:15:29.572775+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-13T14:15:45.599510+0000","last_change":"2024-09-13T14:15:45.599510+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-13T14:15:45.599510+0000","last_undegraded":"2024-09-13T14:15:45.599510+0000","last_fullsized":"2024-09-13T14:15:45.599510+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:17:32.250877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.570594+0000","last_change":"2024-09-13T14:15:22.550265+0000","last_active":"2024-09-13T14:15:29.570594+0000","last_peered":"2024-09-13T14:15:29.570594+0000","last_clean":"2024-09-13T14:15:29.570594+0000","last_became_active":"2024-09-13T14:15:22.549179+0000","last_became_peered":"2024-09-13T14:15:22.549179+0000","last_unstale":"2024-09-13T14:15:29.570594+0000","last_undegraded":"2024-09-13T14:15:29.570594+0000","last_fullsized":"2024-09-13T14:15:29.570594+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T14:15:29.565710+0000","last_change":"2024-09-13T14:15:26.575732+0000","last_active":"2024-09-13T14:15:29.565710+0000","last_peered":"2024-09-13T14:15:29.565710+0000","last_clean":"2024-09-13T14:15:29.565710+0000","last_became_active":"2024-09-13T14:15:26.575235+0000","last_became_peered":"2024-09-13T14:15:26.575235+0000","last_unstale":"2024-09-13T14:15:29.565710+0000","last_undegraded":"2024-09-13T14:15:29.565710+0000","last_fullsized":"2024-09-13T14:15:29.565710+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597924+0000","last_change":"2024-09-13T14:15:22.542901+0000","last_active":"2024-09-13T14:15:45.597924+0000","last_peered":"2024-09-13T14:15:45.597924+0000","last_clean":"2024-09-13T14:15:45.597924+0000","last_became_active":"2024-09-13T14:15:22.542604+0000","last_became_peered":"2024-09-13T14:15:22.542604+0000","last_unstale":"2024-09-13T14:15:45.597924+0000","last_undegraded":"2024-09-13T14:15:45.597924+0000","last_fullsized":"2024-09-13T14:15:45.597924+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597651+0000","last_change":"2024-09-13T14:15:26.555625+0000","last_active":"2024-09-13T14:15:45.597651+0000","last_peered":"2024-09-13T14:15:45.597651+0000","last_clean":"2024-09-13T14:15:45.597651+0000","last_became_active":"2024-09-13T14:15:26.554633+0000","last_became_peered":"2024-09-13T14:15:26.554633+0000","last_unstale":"2024-09-13T14:15:45.597651+0000","last_undegraded":"2024-09-13T14:15:45.597651+0000","last_fullsized":"2024-09-13T14:15:45.597651+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":21,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-13T14:15:45.600791+0000","last_change":"2024-09-13T14:15:45.601037+0000","last_active":"2024-09-13T14:15:45.600763+0000","last_peered":"2024-09-13T14:15:29.571584+0000","last_clean":"2024-09-13T14:15:29.571584+0000","last_became_active":"2024-09-13T14:15:22.540722+0000","last_became_peered":"2024-09-13T14:15:22.540722+0000","last_unstale":"2024-09-13T14:15:45.600791+0000","last_undegraded":"2024-09-13T14:15:45.600791+0000","last_fullsized":"2024-09-13T14:15:45.600791+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.342032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430221,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":835486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.749}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56999999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42499999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71799999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53400000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626449,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.58499999999999996}]},{"osd":1,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.46700000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.57699999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822677,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":252100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.60999999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.63300000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.65500000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.5}]},{"osd":4,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.42399999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:15:31 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.41099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018906,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.45600000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82999999999999996,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.63100000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56999999999999995,"15min":0.56399999999999995},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78400000000000003,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.443}]},{"osd":2,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.85299999999999998,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41799999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56100000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88200000000000001,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.56999999999999995},"min":{"1min":0.46800000000000003,"5min":0.374,"15min":0.374},"max":{"1min":0.81699999999999995,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47099999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:15:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.48299999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.5}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":246182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.53200000000000003,"5min":0.628,"15min":0.628},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.222}]},{"osd":1,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55000000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.28499999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.24099999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:14:43 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68899999999999995}]},{"osd":5,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.69699999999999995}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.67600000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":552,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":565248,"data_stored":237793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.379,"15min":0.374},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.432,"5min":0.39300000000000002,"15min":0.38700000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.28100000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:15:02 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.41099999999999998,"15min":0.40600000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60199999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.40899999999999997,"15min":0.40300000000000002},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.47899999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:15:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.60599999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:15:44 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.59399999999999997,"15min":0.58899999999999997},"min":{"1min":0.48399999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.58899999999999997,"15min":0.58399999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.54000000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:15:07 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52800000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83099999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":244056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.30099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60999999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.40699999999999997,"15min":0.37},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.33300000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:15:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.40500000000000003,"15min":0.39000000000000001},"min":{"1min":0.28999999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.29499999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:15:25 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.42999999999999999,"15min":0.42399999999999999},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.374}]},{"osd":4,"last update":"Fri Sep 13 14:14:43 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.59699999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:15:10 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.61199999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:15:35 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.56100000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67500000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1187840,"data_stored":835610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.34200000000000003},"min":{"1min":0.26100000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":0.51600000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.35799999999999998,"15min":0.34499999999999997},"min":{"1min":0.251,"5min":0.184,"15min":0.184},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.31}]},{"osd":2,"last update":"Fri Sep 13 14:15:01 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.29399999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.373,"15min":0.36599999999999999},"min":{"1min":0.29399999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.32800000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:15:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.34999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.442},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.61799999999999999,"5min":0.625,"15min":0.625},"last":0.38500000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:15:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.39300000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.46700000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:15:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.375}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.39700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:15:51.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:15:51.403 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:51.786 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:51 smithi026 ceph-mon[27770]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:51.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:51 smithi026 ceph-mon[39182]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:51 smithi145 ceph-mon[31057]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:52 smithi026 ceph-mon[27770]: pgmap v202: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:52.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:52 smithi026 ceph-mon[39182]: pgmap v202: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:52.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:52 smithi145 ceph-mon[31057]: pgmap v202: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:53.446 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:15:53.448 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-09-13T14:15:54.250 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-13T14:15:53.262024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":590938,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385280,"kb_used_data":6424,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575936,"statfs":{"total":767960285184,"available":765517758464,"internally_reserved":0,"allocated":6578176,"data_stored":3785268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001669"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603379+0000","last_change":"2024-09-13T14:15:26.568393+0000","last_active":"2024-09-13T14:15:45.603379+0000","last_peered":"2024-09-13T14:15:45.603379+0000","last_clean":"2024-09-13T14:15:45.603379+0000","last_became_active":"2024-09-13T14:15:26.568075+0000","last_became_peered":"2024-09-13T14:15:26.568075+0000","last_unstale":"2024-09-13T14:15:45.603379+0000","last_undegraded":"2024-09-13T14:15:45.603379+0000","last_fullsized":"2024-09-13T14:15:45.603379+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602211+0000","last_change":"2024-09-13T14:15:26.563624+0000","last_active":"2024-09-13T14:15:45.602211+0000","last_peered":"2024-09-13T14:15:45.602211+0000","last_clean":"2024-09-13T14:15:45.602211+0000","last_became_active":"2024-09-13T14:15:26.563364+0000","last_became_peered":"2024-09-13T14:15:26.563364+0000","last_unstale":"2024-09-13T14:15:45.602211+0000","last_undegraded":"2024-09-13T14:15:45.602211+0000","last_fullsized":"2024-09-13T14:15:45.602211+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.600214+0000","last_change":"2024-09-13T14:15:26.566586+0000","last_active":"2024-09-13T14:15:45.600214+0000","last_peered":"2024-09-13T14:15:45.600214+0000","last_clean":"2024-09-13T14:15:45.600214+0000","last_became_active":"2024-09-13T14:15:26.566331+0000","last_became_peered":"2024-09-13T14:15:26.566331+0000","last_unstale":"2024-09-13T14:15:45.600214+0000","last_undegraded":"2024-09-13T14:15:45.600214+0000","last_fullsized":"2024-09-13T14:15:45.600214+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601352+0000","last_change":"2024-09-13T14:15:26.564769+0000","last_active":"2024-09-13T14:15:45.601352+0000","last_peered":"2024-09-13T14:15:45.601352+0000","last_clean":"2024-09-13T14:15:45.601352+0000","last_became_active":"2024-09-13T14:15:26.564496+0000","last_became_peered":"2024-09-13T14:15:26.564496+0000","last_unstale":"2024-09-13T14:15:45.601352+0000","last_undegraded":"2024-09-13T14:15:45.601352+0000","last_fullsized":"2024-09-13T14:15:45.601352+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603182+0000","last_change":"2024-09-13T14:15:26.555592+0000","last_active":"2024-09-13T14:15:45.603182+0000","last_peered":"2024-09-13T14:15:45.603182+0000","last_clean":"2024-09-13T14:15:45.603182+0000","last_became_active":"2024-09-13T14:15:26.555311+0000","last_became_peered":"2024-09-13T14:15:26.555311+0000","last_unstale":"2024-09-13T14:15:45.603182+0000","last_undegraded":"2024-09-13T14:15:45.603182+0000","last_fullsized":"2024-09-13T14:15:45.603182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601228+0000","last_change":"2024-09-13T14:15:26.572204+0000","last_active":"2024-09-13T14:15:45.601228+0000","last_peered":"2024-09-13T14:15:45.601228+0000","last_clean":"2024-09-13T14:15:45.601228+0000","last_became_active":"2024-09-13T14:15:26.571310+0000","last_became_peered":"2024-09-13T14:15:26.571310+0000","last_unstale":"2024-09-13T14:15:45.601228+0000","last_undegraded":"2024-09-13T14:15:45.601228+0000","last_fullsized":"2024-09-13T14:15:45.601228+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602683+0000","last_change":"2024-09-13T14:15:26.566676+0000","last_active":"2024-09-13T14:15:45.602683+0000","last_peered":"2024-09-13T14:15:45.602683+0000","last_clean":"2024-09-13T14:15:45.602683+0000","last_became_active":"2024-09-13T14:15:26.566403+0000","last_became_peered":"2024-09-13T14:15:26.566403+0000","last_unstale":"2024-09-13T14:15:45.602683+0000","last_undegraded":"2024-09-13T14:15:45.602683+0000","last_fullsized":"2024-09-13T14:15:45.602683+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596840+0000","last_change":"2024-09-13T14:15:26.566663+0000","last_active":"2024-09-13T14:15:45.596840+0000","last_peered":"2024-09-13T14:15:45.596840+0000","last_clean":"2024-09-13T14:15:45.596840+0000","last_became_active":"2024-09-13T14:15:26.566400+0000","last_became_peered":"2024-09-13T14:15:26.566400+0000","last_unstale":"2024-09-13T14:15:45.596840+0000","last_undegraded":"2024-09-13T14:15:45.596840+0000","last_fullsized":"2024-09-13T14:15:45.596840+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603399+0000","last_change":"2024-09-13T14:15:26.570338+0000","last_active":"2024-09-13T14:15:45.603399+0000","last_peered":"2024-09-13T14:15:45.603399+0000","last_clean":"2024-09-13T14:15:45.603399+0000","last_became_active":"2024-09-13T14:15:26.569776+0000","last_became_peered":"2024-09-13T14:15:26.569776+0000","last_unstale":"2024-09-13T14:15:45.603399+0000","last_undegraded":"2024-09-13T14:15:45.603399+0000","last_fullsized":"2024-09-13T14:15:45.603399+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044993+0000","last_change":"2024-09-13T14:15:26.564796+0000","last_active":"2024-09-13T14:15:46.044993+0000","last_peered":"2024-09-13T14:15:46.044993+0000","last_clean":"2024-09-13T14:15:46.044993+0000","last_became_active":"2024-09-13T14:15:26.563959+0000","last_became_peered":"2024-09-13T14:15:26.563959+0000","last_unstale":"2024-09-13T14:15:46.044993+0000","last_undegraded":"2024-09-13T14:15:46.044993+0000","last_fullsized":"2024-09-13T14:15:46.044993+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044871+0000","last_change":"2024-09-13T14:15:26.567040+0000","last_active":"2024-09-13T14:15:46.044871+0000","last_peered":"2024-09-13T14:15:46.044871+0000","last_clean":"2024-09-13T14:15:46.044871+0000","last_became_active":"2024-09-13T14:15:26.566726+0000","last_became_peered":"2024-09-13T14:15:26.566726+0000","last_unstale":"2024-09-13T14:15:46.044871+0000","last_undegraded":"2024-09-13T14:15:46.044871+0000","last_fullsized":"2024-09-13T14:15:46.044871+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598208+0000","last_change":"2024-09-13T14:15:26.563679+0000","last_active":"2024-09-13T14:15:45.598208+0000","last_peered":"2024-09-13T14:15:45.598208+0000","last_clean":"2024-09-13T14:15:45.598208+0000","last_became_active":"2024-09-13T14:15:26.563402+0000","last_became_peered":"2024-09-13T14:15:26.563402+0000","last_unstale":"2024-09-13T14:15:45.598208+0000","last_undegraded":"2024-09-13T14:15:45.598208+0000","last_fullsized":"2024-09-13T14:15:45.598208+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597444+0000","last_change":"2024-09-13T14:15:26.552634+0000","last_active":"2024-09-13T14:15:45.597444+0000","last_peered":"2024-09-13T14:15:45.597444+0000","last_clean":"2024-09-13T14:15:45.597444+0000","last_became_active":"2024-09-13T14:15:26.552354+0000","last_became_peered":"2024-09-13T14:15:26.552354+0000","last_unstale":"2024-09-13T14:15:45.597444+0000","last_undegraded":"2024-09-13T14:15:45.597444+0000","last_fullsized":"2024-09-13T14:15:45.597444+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.599881+0000","last_change":"2024-09-13T14:15:26.576136+0000","last_active":"2024-09-13T14:15:45.599881+0000","last_peered":"2024-09-13T14:15:45.599881+0000","last_clean":"2024-09-13T14:15:45.599881+0000","last_became_active":"2024-09-13T14:15:26.575737+0000","last_became_peered":"2024-09-13T14:15:26.575737+0000","last_unstale":"2024-09-13T14:15:45.599881+0000","last_undegraded":"2024-09-13T14:15:45.599881+0000","last_fullsized":"2024-09-13T14:15:45.599881+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601543+0000","last_change":"2024-09-13T14:15:26.566237+0000","last_active":"2024-09-13T14:15:45.601543+0000","last_peered":"2024-09-13T14:15:45.601543+0000","last_clean":"2024-09-13T14:15:45.601543+0000","last_became_active":"2024-09-13T14:15:26.565645+0000","last_became_peered":"2024-09-13T14:15:26.565645+0000","last_unstale":"2024-09-13T14:15:45.601543+0000","last_undegraded":"2024-09-13T14:15:45.601543+0000","last_fullsized":"2024-09-13T14:15:45.601543+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.965107+0000","last_change":"2024-09-13T14:15:46.965107+0000","last_active":"2024-09-13T14:15:46.965107+0000","last_peered":"2024-09-13T14:15:46.965107+0000","last_clean":"2024-09-13T14:15:46.965107+0000","last_became_active":"2024-09-13T14:15:46.964708+0000","last_became_peered":"2024-09-13T14:15:46.964708+0000","last_unstale":"2024-09-13T14:15:46.965107+0000","last_undegraded":"2024-09-13T14:15:46.965107+0000","last_fullsized":"2024-09-13T14:15:46.965107+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601904+0000","last_change":"2024-09-13T14:15:26.565535+0000","last_active":"2024-09-13T14:15:45.601904+0000","last_peered":"2024-09-13T14:15:45.601904+0000","last_clean":"2024-09-13T14:15:45.601904+0000","last_became_active":"2024-09-13T14:15:26.565091+0000","last_became_peered":"2024-09-13T14:15:26.565091+0000","last_unstale":"2024-09-13T14:15:45.601904+0000","last_undegraded":"2024-09-13T14:15:45.601904+0000","last_fullsized":"2024-09-13T14:15:45.601904+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598066+0000","last_change":"2024-09-13T14:15:26.556686+0000","last_active":"2024-09-13T14:15:45.598066+0000","last_peered":"2024-09-13T14:15:45.598066+0000","last_clean":"2024-09-13T14:15:45.598066+0000","last_became_active":"2024-09-13T14:15:26.556349+0000","last_became_peered":"2024-09-13T14:15:26.556349+0000","last_unstale":"2024-09-13T14:15:45.598066+0000","last_undegraded":"2024-09-13T14:15:45.598066+0000","last_fullsized":"2024-09-13T14:15:45.598066+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603149+0000","last_change":"2024-09-13T14:15:26.564825+0000","last_active":"2024-09-13T14:15:45.603149+0000","last_peered":"2024-09-13T14:15:45.603149+0000","last_clean":"2024-09-13T14:15:45.603149+0000","last_became_active":"2024-09-13T14:15:26.563703+0000","last_became_peered":"2024-09-13T14:15:26.563703+0000","last_unstale":"2024-09-13T14:15:45.603149+0000","last_undegraded":"2024-09-13T14:15:45.603149+0000","last_fullsized":"2024-09-13T14:15:45.603149+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044809+0000","last_change":"2024-09-13T14:15:26.568327+0000","last_active":"2024-09-13T14:15:46.044809+0000","last_peered":"2024-09-13T14:15:46.044809+0000","last_clean":"2024-09-13T14:15:46.044809+0000","last_became_active":"2024-09-13T14:15:26.568070+0000","last_became_peered":"2024-09-13T14:15:26.568070+0000","last_unstale":"2024-09-13T14:15:46.044809+0000","last_undegraded":"2024-09-13T14:15:46.044809+0000","last_fullsized":"2024-09-13T14:15:46.044809+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.600181+0000","last_change":"2024-09-13T14:15:26.568339+0000","last_active":"2024-09-13T14:15:45.600181+0000","last_peered":"2024-09-13T14:15:45.600181+0000","last_clean":"2024-09-13T14:15:45.600181+0000","last_became_active":"2024-09-13T14:15:26.568056+0000","last_became_peered":"2024-09-13T14:15:26.568056+0000","last_unstale":"2024-09-13T14:15:45.600181+0000","last_undegraded":"2024-09-13T14:15:45.600181+0000","last_fullsized":"2024-09-13T14:15:45.600181+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601444+0000","last_change":"2024-09-13T14:15:26.572417+0000","last_active":"2024-09-13T14:15:45.601444+0000","last_peered":"2024-09-13T14:15:45.601444+0000","last_clean":"2024-09-13T14:15:45.601444+0000","last_became_active":"2024-09-13T14:15:26.572126+0000","last_became_peered":"2024-09-13T14:15:26.572126+0000","last_unstale":"2024-09-13T14:15:45.601444+0000","last_undegraded":"2024-09-13T14:15:45.601444+0000","last_fullsized":"2024-09-13T14:15:45.601444+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597141+0000","last_change":"2024-09-13T14:15:26.555819+0000","last_active":"2024-09-13T14:15:45.597141+0000","last_peered":"2024-09-13T14:15:45.597141+0000","last_clean":"2024-09-13T14:15:45.597141+0000","last_became_active":"2024-09-13T14:15:26.555172+0000","last_became_peered":"2024-09-13T14:15:26.555172+0000","last_unstale":"2024-09-13T14:15:45.597141+0000","last_undegraded":"2024-09-13T14:15:45.597141+0000","last_fullsized":"2024-09-13T14:15:45.597141+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601257+0000","last_change":"2024-09-13T14:15:26.561070+0000","last_active":"2024-09-13T14:15:45.601257+0000","last_peered":"2024-09-13T14:15:45.601257+0000","last_clean":"2024-09-13T14:15:45.601257+0000","last_became_active":"2024-09-13T14:15:26.559771+0000","last_became_peered":"2024-09-13T14:15:26.559771+0000","last_unstale":"2024-09-13T14:15:45.601257+0000","last_undegraded":"2024-09-13T14:15:45.601257+0000","last_fullsized":"2024-09-13T14:15:45.601257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597763+0000","last_change":"2024-09-13T14:15:26.554841+0000","last_active":"2024-09-13T14:15:45.597763+0000","last_peered":"2024-09-13T14:15:45.597763+0000","last_clean":"2024-09-13T14:15:45.597763+0000","last_became_active":"2024-09-13T14:15:26.553231+0000","last_became_peered":"2024-09-13T14:15:26.553231+0000","last_unstale":"2024-09-13T14:15:45.597763+0000","last_undegraded":"2024-09-13T14:15:45.597763+0000","last_fullsized":"2024-09-13T14:15:45.597763+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045061+0000","last_change":"2024-09-13T14:15:26.551060+0000","last_active":"2024-09-13T14:15:46.045061+0000","last_peered":"2024-09-13T14:15:46.045061+0000","last_clean":"2024-09-13T14:15:46.045061+0000","last_became_active":"2024-09-13T14:15:26.550739+0000","last_became_peered":"2024-09-13T14:15:26.550739+0000","last_unstale":"2024-09-13T14:15:46.045061+0000","last_undegraded":"2024-09-13T14:15:46.045061+0000","last_fullsized":"2024-09-13T14:15:46.045061+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601851+0000","last_change":"2024-09-13T14:15:26.559670+0000","last_active":"2024-09-13T14:15:45.601851+0000","last_peered":"2024-09-13T14:15:45.601851+0000","last_clean":"2024-09-13T14:15:45.601851+0000","last_became_active":"2024-09-13T14:15:26.559401+0000","last_became_peered":"2024-09-13T14:15:26.559401+0000","last_unstale":"2024-09-13T14:15:45.601851+0000","last_undegraded":"2024-09-13T14:15:45.601851+0000","last_fullsized":"2024-09-13T14:15:45.601851+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597525+0000","last_change":"2024-09-13T14:15:26.570832+0000","last_active":"2024-09-13T14:15:45.597525+0000","last_peered":"2024-09-13T14:15:45.597525+0000","last_clean":"2024-09-13T14:15:45.597525+0000","last_became_active":"2024-09-13T14:15:26.570531+0000","last_became_peered":"2024-09-13T14:15:26.570531+0000","last_unstale":"2024-09-13T14:15:45.597525+0000","last_undegraded":"2024-09-13T14:15:45.597525+0000","last_fullsized":"2024-09-13T14:15:45.597525+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.600262+0000","last_change":"2024-09-13T14:15:26.573053+0000","last_active":"2024-09-13T14:15:45.600262+0000","last_peered":"2024-09-13T14:15:45.600262+0000","last_clean":"2024-09-13T14:15:45.600262+0000","last_became_active":"2024-09-13T14:15:26.572783+0000","last_became_peered":"2024-09-13T14:15:26.572783+0000","last_unstale":"2024-09-13T14:15:45.600262+0000","last_undegraded":"2024-09-13T14:15:45.600262+0000","last_fullsized":"2024-09-13T14:15:45.600262+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601363+0000","last_change":"2024-09-13T14:15:26.569346+0000","last_active":"2024-09-13T14:15:45.601363+0000","last_peered":"2024-09-13T14:15:45.601363+0000","last_clean":"2024-09-13T14:15:45.601363+0000","last_became_active":"2024-09-13T14:15:26.568939+0000","last_became_peered":"2024-09-13T14:15:26.568939+0000","last_unstale":"2024-09-13T14:15:45.601363+0000","last_undegraded":"2024-09-13T14:15:45.601363+0000","last_fullsized":"2024-09-13T14:15:45.601363+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596906+0000","last_change":"2024-09-13T14:15:26.562543+0000","last_active":"2024-09-13T14:15:45.596906+0000","last_peered":"2024-09-13T14:15:45.596906+0000","last_clean":"2024-09-13T14:15:45.596906+0000","last_became_active":"2024-09-13T14:15:26.562267+0000","last_became_peered":"2024-09-13T14:15:26.562267+0000","last_unstale":"2024-09-13T14:15:45.596906+0000","last_undegraded":"2024-09-13T14:15:45.596906+0000","last_fullsized":"2024-09-13T14:15:45.596906+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044457+0000","last_change":"2024-09-13T14:15:26.568440+0000","last_active":"2024-09-13T14:15:46.044457+0000","last_peered":"2024-09-13T14:15:46.044457+0000","last_clean":"2024-09-13T14:15:46.044457+0000","last_became_active":"2024-09-13T14:15:26.568169+0000","last_became_peered":"2024-09-13T14:15:26.568169+0000","last_unstale":"2024-09-13T14:15:46.044457+0000","last_undegraded":"2024-09-13T14:15:46.044457+0000","last_fullsized":"2024-09-13T14:15:46.044457+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.963486+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:15:46.963486+0000","last_peered":"2024-09-13T14:15:46.963486+0000","last_clean":"2024-09-13T14:15:46.963486+0000","last_became_active":"2024-09-13T14:15:46.963029+0000","last_became_peered":"2024-09-13T14:15:46.963029+0000","last_unstale":"2024-09-13T14:15:46.963486+0000","last_undegraded":"2024-09-13T14:15:46.963486+0000","last_fullsized":"2024-09-13T14:15:46.963486+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597677+0000","last_change":"2024-09-13T14:15:22.541205+0000","last_active":"2024-09-13T14:15:45.597677+0000","last_peered":"2024-09-13T14:15:45.597677+0000","last_clean":"2024-09-13T14:15:45.597677+0000","last_became_active":"2024-09-13T14:15:22.539574+0000","last_became_peered":"2024-09-13T14:15:22.539574+0000","last_unstale":"2024-09-13T14:15:45.597677+0000","last_undegraded":"2024-09-13T14:15:45.597677+0000","last_fullsized":"2024-09-13T14:15:45.597677+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.606065+0000","last_change":"2024-09-13T14:15:46.606065+0000","last_active":"2024-09-13T14:15:46.606065+0000","last_peered":"2024-09-13T14:15:46.606065+0000","last_clean":"2024-09-13T14:15:46.606065+0000","last_became_active":"2024-09-13T14:15:46.605583+0000","last_became_peered":"2024-09-13T14:15:46.605583+0000","last_unstale":"2024-09-13T14:15:46.606065+0000","last_undegraded":"2024-09-13T14:15:46.606065+0000","last_fullsized":"2024-09-13T14:15:46.606065+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:21.907855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044687+0000","last_change":"2024-09-13T14:15:22.543444+0000","last_active":"2024-09-13T14:15:46.044687+0000","last_peered":"2024-09-13T14:15:46.044687+0000","last_clean":"2024-09-13T14:15:46.044687+0000","last_became_active":"2024-09-13T14:15:22.543138+0000","last_became_peered":"2024-09-13T14:15:22.543138+0000","last_unstale":"2024-09-13T14:15:46.044687+0000","last_undegraded":"2024-09-13T14:15:46.044687+0000","last_fullsized":"2024-09-13T14:15:46.044687+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597273+0000","last_change":"2024-09-13T14:15:22.541457+0000","last_active":"2024-09-13T14:15:45.597273+0000","last_peered":"2024-09-13T14:15:45.597273+0000","last_clean":"2024-09-13T14:15:45.597273+0000","last_became_active":"2024-09-13T14:15:22.539753+0000","last_became_peered":"2024-09-13T14:15:22.539753+0000","last_unstale":"2024-09-13T14:15:45.597273+0000","last_undegraded":"2024-09-13T14:15:45.597273+0000","last_fullsized":"2024-09-13T14:15:45.597273+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596847+0000","last_change":"2024-09-13T14:15:26.571137+0000","last_active":"2024-09-13T14:15:45.596847+0000","last_peered":"2024-09-13T14:15:45.596847+0000","last_clean":"2024-09-13T14:15:45.596847+0000","last_became_active":"2024-09-13T14:15:26.570884+0000","last_became_peered":"2024-09-13T14:15:26.570884+0000","last_unstale":"2024-09-13T14:15:45.596847+0000","last_undegraded":"2024-09-13T14:15:45.596847+0000","last_fullsized":"2024-09-13T14:15:45.596847+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601578+0000","last_change":"2024-09-13T14:15:22.546875+0000","last_active":"2024-09-13T14:15:45.601578+0000","last_peered":"2024-09-13T14:15:45.601578+0000","last_clean":"2024-09-13T14:15:45.601578+0000","last_became_active":"2024-09-13T14:15:22.545863+0000","last_became_peered":"2024-09-13T14:15:22.545863+0000","last_unstale":"2024-09-13T14:15:45.601578+0000","last_undegraded":"2024-09-13T14:15:45.601578+0000","last_fullsized":"2024-09-13T14:15:45.601578+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601347+0000","last_change":"2024-09-13T14:15:26.560575+0000","last_active":"2024-09-13T14:15:45.601347+0000","last_peered":"2024-09-13T14:15:45.601347+0000","last_clean":"2024-09-13T14:15:45.601347+0000","last_became_active":"2024-09-13T14:15:26.560291+0000","last_became_peered":"2024-09-13T14:15:26.560291+0000","last_unstale":"2024-09-13T14:15:45.601347+0000","last_undegraded":"2024-09-13T14:15:45.601347+0000","last_fullsized":"2024-09-13T14:15:45.601347+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045029+0000","last_change":"2024-09-13T14:15:22.546137+0000","last_active":"2024-09-13T14:15:46.045029+0000","last_peered":"2024-09-13T14:15:46.045029+0000","last_clean":"2024-09-13T14:15:46.045029+0000","last_became_active":"2024-09-13T14:15:22.545833+0000","last_became_peered":"2024-09-13T14:15:22.545833+0000","last_unstale":"2024-09-13T14:15:46.045029+0000","last_undegraded":"2024-09-13T14:15:46.045029+0000","last_fullsized":"2024-09-13T14:15:46.045029+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602112+0000","last_change":"2024-09-13T14:15:26.566451+0000","last_active":"2024-09-13T14:15:45.602112+0000","last_peered":"2024-09-13T14:15:45.602112+0000","last_clean":"2024-09-13T14:15:45.602112+0000","last_became_active":"2024-09-13T14:15:26.565363+0000","last_became_peered":"2024-09-13T14:15:26.565363+0000","last_unstale":"2024-09-13T14:15:45.602112+0000","last_undegraded":"2024-09-13T14:15:45.602112+0000","last_fullsized":"2024-09-13T14:15:45.602112+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602192+0000","last_change":"2024-09-13T14:15:22.540966+0000","last_active":"2024-09-13T14:15:45.602192+0000","last_peered":"2024-09-13T14:15:45.602192+0000","last_clean":"2024-09-13T14:15:45.602192+0000","last_became_active":"2024-09-13T14:15:22.540611+0000","last_became_peered":"2024-09-13T14:15:22.540611+0000","last_unstale":"2024-09-13T14:15:45.602192+0000","last_undegraded":"2024-09-13T14:15:45.602192+0000","last_fullsized":"2024-09-13T14:15:45.602192+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.599983+0000","last_change":"2024-09-13T14:15:26.573426+0000","last_active":"2024-09-13T14:15:45.599983+0000","last_peered":"2024-09-13T14:15:45.599983+0000","last_clean":"2024-09-13T14:15:45.599983+0000","last_became_active":"2024-09-13T14:15:26.573138+0000","last_became_peered":"2024-09-13T14:15:26.573138+0000","last_unstale":"2024-09-13T14:15:45.599983+0000","last_undegraded":"2024-09-13T14:15:45.599983+0000","last_fullsized":"2024-09-13T14:15:45.599983+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602998+0000","last_change":"2024-09-13T14:15:22.543031+0000","last_active":"2024-09-13T14:15:45.602998+0000","last_peered":"2024-09-13T14:15:45.602998+0000","last_clean":"2024-09-13T14:15:45.602998+0000","last_became_active":"2024-09-13T14:15:22.542739+0000","last_became_peered":"2024-09-13T14:15:22.542739+0000","last_unstale":"2024-09-13T14:15:45.602998+0000","last_undegraded":"2024-09-13T14:15:45.602998+0000","last_fullsized":"2024-09-13T14:15:45.602998+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598301+0000","last_change":"2024-09-13T14:15:26.551575+0000","last_active":"2024-09-13T14:15:45.598301+0000","last_peered":"2024-09-13T14:15:45.598301+0000","last_clean":"2024-09-13T14:15:45.598301+0000","last_became_active":"2024-09-13T14:15:26.550913+0000","last_became_peered":"2024-09-13T14:15:26.550913+0000","last_unstale":"2024-09-13T14:15:45.598301+0000","last_undegraded":"2024-09-13T14:15:45.598301+0000","last_fullsized":"2024-09-13T14:15:45.598301+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044871+0000","last_change":"2024-09-13T14:15:22.539191+0000","last_active":"2024-09-13T14:15:46.044871+0000","last_peered":"2024-09-13T14:15:46.044871+0000","last_clean":"2024-09-13T14:15:46.044871+0000","last_became_active":"2024-09-13T14:15:22.538864+0000","last_became_peered":"2024-09-13T14:15:22.538864+0000","last_unstale":"2024-09-13T14:15:46.044871+0000","last_undegraded":"2024-09-13T14:15:46.044871+0000","last_fullsized":"2024-09-13T14:15:46.044871+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602828+0000","last_change":"2024-09-13T14:15:26.567770+0000","last_active":"2024-09-13T14:15:45.602828+0000","last_peered":"2024-09-13T14:15:45.602828+0000","last_clean":"2024-09-13T14:15:45.602828+0000","last_became_active":"2024-09-13T14:15:26.567513+0000","last_became_peered":"2024-09-13T14:15:26.567513+0000","last_unstale":"2024-09-13T14:15:45.602828+0000","last_undegraded":"2024-09-13T14:15:45.602828+0000","last_fullsized":"2024-09-13T14:15:45.602828+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.600947+0000","last_change":"2024-09-13T14:15:22.545520+0000","last_active":"2024-09-13T14:15:45.600947+0000","last_peered":"2024-09-13T14:15:45.600947+0000","last_clean":"2024-09-13T14:15:45.600947+0000","last_became_active":"2024-09-13T14:15:22.545135+0000","last_became_peered":"2024-09-13T14:15:22.545135+0000","last_unstale":"2024-09-13T14:15:45.600947+0000","last_undegraded":"2024-09-13T14:15:45.600947+0000","last_fullsized":"2024-09-13T14:15:45.600947+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.599965+0000","last_change":"2024-09-13T14:15:26.575023+0000","last_active":"2024-09-13T14:15:45.599965+0000","last_peered":"2024-09-13T14:15:45.599965+0000","last_clean":"2024-09-13T14:15:45.599965+0000","last_became_active":"2024-09-13T14:15:26.574648+0000","last_became_peered":"2024-09-13T14:15:26.574648+0000","last_unstale":"2024-09-13T14:15:45.599965+0000","last_undegraded":"2024-09-13T14:15:45.599965+0000","last_fullsized":"2024-09-13T14:15:45.599965+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601454+0000","last_change":"2024-09-13T14:15:22.546988+0000","last_active":"2024-09-13T14:15:45.601454+0000","last_peered":"2024-09-13T14:15:45.601454+0000","last_clean":"2024-09-13T14:15:45.601454+0000","last_became_active":"2024-09-13T14:15:22.545255+0000","last_became_peered":"2024-09-13T14:15:22.545255+0000","last_unstale":"2024-09-13T14:15:45.601454+0000","last_undegraded":"2024-09-13T14:15:45.601454+0000","last_fullsized":"2024-09-13T14:15:45.601454+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603328+0000","last_change":"2024-09-13T14:15:26.555682+0000","last_active":"2024-09-13T14:15:45.603328+0000","last_peered":"2024-09-13T14:15:45.603328+0000","last_clean":"2024-09-13T14:15:45.603328+0000","last_became_active":"2024-09-13T14:15:26.555391+0000","last_became_peered":"2024-09-13T14:15:26.555391+0000","last_unstale":"2024-09-13T14:15:45.603328+0000","last_undegraded":"2024-09-13T14:15:45.603328+0000","last_fullsized":"2024-09-13T14:15:45.603328+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044543+0000","last_change":"2024-09-13T14:15:22.542050+0000","last_active":"2024-09-13T14:15:46.044543+0000","last_peered":"2024-09-13T14:15:46.044543+0000","last_clean":"2024-09-13T14:15:46.044543+0000","last_became_active":"2024-09-13T14:15:22.541726+0000","last_became_peered":"2024-09-13T14:15:22.541726+0000","last_unstale":"2024-09-13T14:15:46.044543+0000","last_undegraded":"2024-09-13T14:15:46.044543+0000","last_fullsized":"2024-09-13T14:15:46.044543+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044523+0000","last_change":"2024-09-13T14:15:26.567258+0000","last_active":"2024-09-13T14:15:46.044523+0000","last_peered":"2024-09-13T14:15:46.044523+0000","last_clean":"2024-09-13T14:15:46.044523+0000","last_became_active":"2024-09-13T14:15:26.566924+0000","last_became_peered":"2024-09-13T14:15:26.566924+0000","last_unstale":"2024-09-13T14:15:46.044523+0000","last_undegraded":"2024-09-13T14:15:46.044523+0000","last_fullsized":"2024-09-13T14:15:46.044523+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596891+0000","last_change":"2024-09-13T14:15:22.541058+0000","last_active":"2024-09-13T14:15:45.596891+0000","last_peered":"2024-09-13T14:15:45.596891+0000","last_clean":"2024-09-13T14:15:45.596891+0000","last_became_active":"2024-09-13T14:15:22.540793+0000","last_became_peered":"2024-09-13T14:15:22.540793+0000","last_unstale":"2024-09-13T14:15:45.596891+0000","last_undegraded":"2024-09-13T14:15:45.596891+0000","last_fullsized":"2024-09-13T14:15:45.596891+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602765+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:15:45.602765+0000","last_peered":"2024-09-13T14:15:45.602765+0000","last_clean":"2024-09-13T14:15:45.602765+0000","last_became_active":"2024-09-13T14:15:26.562669+0000","last_became_peered":"2024-09-13T14:15:26.562669+0000","last_unstale":"2024-09-13T14:15:45.602765+0000","last_undegraded":"2024-09-13T14:15:45.602765+0000","last_fullsized":"2024-09-13T14:15:45.602765+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598460+0000","last_change":"2024-09-13T14:15:22.540681+0000","last_active":"2024-09-13T14:15:45.598460+0000","last_peered":"2024-09-13T14:15:45.598460+0000","last_clean":"2024-09-13T14:15:45.598460+0000","last_became_active":"2024-09-13T14:15:22.540386+0000","last_became_peered":"2024-09-13T14:15:22.540386+0000","last_unstale":"2024-09-13T14:15:45.598460+0000","last_undegraded":"2024-09-13T14:15:45.598460+0000","last_fullsized":"2024-09-13T14:15:45.598460+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603017+0000","last_change":"2024-09-13T14:15:26.559281+0000","last_active":"2024-09-13T14:15:45.603017+0000","last_peered":"2024-09-13T14:15:45.603017+0000","last_clean":"2024-09-13T14:15:45.603017+0000","last_became_active":"2024-09-13T14:15:26.558987+0000","last_became_peered":"2024-09-13T14:15:26.558987+0000","last_unstale":"2024-09-13T14:15:45.603017+0000","last_undegraded":"2024-09-13T14:15:45.603017+0000","last_fullsized":"2024-09-13T14:15:45.603017+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598251+0000","last_change":"2024-09-13T14:15:22.544147+0000","last_active":"2024-09-13T14:15:45.598251+0000","last_peered":"2024-09-13T14:15:45.598251+0000","last_clean":"2024-09-13T14:15:45.598251+0000","last_became_active":"2024-09-13T14:15:22.543782+0000","last_became_peered":"2024-09-13T14:15:22.543782+0000","last_unstale":"2024-09-13T14:15:45.598251+0000","last_undegraded":"2024-09-13T14:15:45.598251+0000","last_fullsized":"2024-09-13T14:15:45.598251+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602966+0000","last_change":"2024-09-13T14:15:26.570603+0000","last_active":"2024-09-13T14:15:45.602966+0000","last_peered":"2024-09-13T14:15:45.602966+0000","last_clean":"2024-09-13T14:15:45.602966+0000","last_became_active":"2024-09-13T14:15:26.569357+0000","last_became_peered":"2024-09-13T14:15:26.569357+0000","last_unstale":"2024-09-13T14:15:45.602966+0000","last_undegraded":"2024-09-13T14:15:45.602966+0000","last_fullsized":"2024-09-13T14:15:45.602966+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602001+0000","last_change":"2024-09-13T14:15:22.546363+0000","last_active":"2024-09-13T14:15:45.602001+0000","last_peered":"2024-09-13T14:15:45.602001+0000","last_clean":"2024-09-13T14:15:45.602001+0000","last_became_active":"2024-09-13T14:15:22.544417+0000","last_became_peered":"2024-09-13T14:15:22.544417+0000","last_unstale":"2024-09-13T14:15:45.602001+0000","last_undegraded":"2024-09-13T14:15:45.602001+0000","last_fullsized":"2024-09-13T14:15:45.602001+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601813+0000","last_change":"2024-09-13T14:15:26.572445+0000","last_active":"2024-09-13T14:15:45.601813+0000","last_peered":"2024-09-13T14:15:45.601813+0000","last_clean":"2024-09-13T14:15:45.601813+0000","last_became_active":"2024-09-13T14:15:26.572166+0000","last_became_peered":"2024-09-13T14:15:26.572166+0000","last_unstale":"2024-09-13T14:15:45.601813+0000","last_undegraded":"2024-09-13T14:15:45.601813+0000","last_fullsized":"2024-09-13T14:15:45.601813+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597159+0000","last_change":"2024-09-13T14:15:22.542920+0000","last_active":"2024-09-13T14:15:45.597159+0000","last_peered":"2024-09-13T14:15:45.597159+0000","last_clean":"2024-09-13T14:15:45.597159+0000","last_became_active":"2024-09-13T14:15:22.542641+0000","last_became_peered":"2024-09-13T14:15:22.542641+0000","last_unstale":"2024-09-13T14:15:45.597159+0000","last_undegraded":"2024-09-13T14:15:45.597159+0000","last_fullsized":"2024-09-13T14:15:45.597159+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602997+0000","last_change":"2024-09-13T14:15:26.572195+0000","last_active":"2024-09-13T14:15:45.602997+0000","last_peered":"2024-09-13T14:15:45.602997+0000","last_clean":"2024-09-13T14:15:45.602997+0000","last_became_active":"2024-09-13T14:15:26.571003+0000","last_became_peered":"2024-09-13T14:15:26.571003+0000","last_unstale":"2024-09-13T14:15:45.602997+0000","last_undegraded":"2024-09-13T14:15:45.602997+0000","last_fullsized":"2024-09-13T14:15:45.602997+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602575+0000","last_change":"2024-09-13T14:14:57.479980+0000","last_active":"2024-09-13T14:15:45.602575+0000","last_peered":"2024-09-13T14:15:45.602575+0000","last_clean":"2024-09-13T14:15:45.602575+0000","last_became_active":"2024-09-13T14:14:57.475298+0000","last_became_peered":"2024-09-13T14:14:57.475298+0000","last_unstale":"2024-09-13T14:15:45.602575+0000","last_undegraded":"2024-09-13T14:15:45.602575+0000","last_fullsized":"2024-09-13T14:15:45.602575+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_clean_scrub_stamp":"2024-09-13T14:12:56.059430+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601202+0000","last_change":"2024-09-13T14:15:22.542042+0000","last_active":"2024-09-13T14:15:45.601202+0000","last_peered":"2024-09-13T14:15:45.601202+0000","last_clean":"2024-09-13T14:15:45.601202+0000","last_became_active":"2024-09-13T14:15:22.541756+0000","last_became_peered":"2024-09-13T14:15:22.541756+0000","last_unstale":"2024-09-13T14:15:45.601202+0000","last_undegraded":"2024-09-13T14:15:45.601202+0000","last_fullsized":"2024-09-13T14:15:45.601202+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597402+0000","last_change":"2024-09-13T14:15:26.565510+0000","last_active":"2024-09-13T14:15:45.597402+0000","last_peered":"2024-09-13T14:15:45.597402+0000","last_clean":"2024-09-13T14:15:45.597402+0000","last_became_active":"2024-09-13T14:15:26.565213+0000","last_became_peered":"2024-09-13T14:15:26.565213+0000","last_unstale":"2024-09-13T14:15:45.597402+0000","last_undegraded":"2024-09-13T14:15:45.597402+0000","last_fullsized":"2024-09-13T14:15:45.597402+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602531+0000","last_change":"2024-09-13T14:15:22.550379+0000","last_active":"2024-09-13T14:15:45.602531+0000","last_peered":"2024-09-13T14:15:45.602531+0000","last_clean":"2024-09-13T14:15:45.602531+0000","last_became_active":"2024-09-13T14:15:22.549995+0000","last_became_peered":"2024-09-13T14:15:22.549995+0000","last_unstale":"2024-09-13T14:15:45.602531+0000","last_undegraded":"2024-09-13T14:15:45.602531+0000","last_fullsized":"2024-09-13T14:15:45.602531+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.965011+0000","last_change":"2024-09-13T14:15:46.965011+0000","last_active":"2024-09-13T14:15:46.965011+0000","last_peered":"2024-09-13T14:15:46.965011+0000","last_clean":"2024-09-13T14:15:46.965011+0000","last_became_active":"2024-09-13T14:15:46.964646+0000","last_became_peered":"2024-09-13T14:15:46.964646+0000","last_unstale":"2024-09-13T14:15:46.965011+0000","last_undegraded":"2024-09-13T14:15:46.965011+0000","last_fullsized":"2024-09-13T14:15:46.965011+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:07.136822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597040+0000","last_change":"2024-09-13T14:15:22.544154+0000","last_active":"2024-09-13T14:15:45.597040+0000","last_peered":"2024-09-13T14:15:45.597040+0000","last_clean":"2024-09-13T14:15:45.597040+0000","last_became_active":"2024-09-13T14:15:22.543880+0000","last_became_peered":"2024-09-13T14:15:22.543880+0000","last_unstale":"2024-09-13T14:15:45.597040+0000","last_undegraded":"2024-09-13T14:15:45.597040+0000","last_fullsized":"2024-09-13T14:15:45.597040+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602573+0000","last_change":"2024-09-13T14:15:26.553134+0000","last_active":"2024-09-13T14:15:45.602573+0000","last_peered":"2024-09-13T14:15:45.602573+0000","last_clean":"2024-09-13T14:15:45.602573+0000","last_became_active":"2024-09-13T14:15:26.552054+0000","last_became_peered":"2024-09-13T14:15:26.552054+0000","last_unstale":"2024-09-13T14:15:45.602573+0000","last_undegraded":"2024-09-13T14:15:45.602573+0000","last_fullsized":"2024-09-13T14:15:45.602573+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601553+0000","last_change":"2024-09-13T14:15:22.545901+0000","last_active":"2024-09-13T14:15:45.601553+0000","last_peered":"2024-09-13T14:15:45.601553+0000","last_clean":"2024-09-13T14:15:45.601553+0000","last_became_active":"2024-09-13T14:15:22.545610+0000","last_became_peered":"2024-09-13T14:15:22.545610+0000","last_unstale":"2024-09-13T14:15:45.601553+0000","last_undegraded":"2024-09-13T14:15:45.601553+0000","last_fullsized":"2024-09-13T14:15:45.601553+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598680+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:15:45.598680+0000","last_peered":"2024-09-13T14:15:45.598680+0000","last_clean":"2024-09-13T14:15:45.598680+0000","last_became_active":"2024-09-13T14:15:26.562679+0000","last_became_peered":"2024-09-13T14:15:26.562679+0000","last_unstale":"2024-09-13T14:15:45.598680+0000","last_undegraded":"2024-09-13T14:15:45.598680+0000","last_fullsized":"2024-09-13T14:15:45.598680+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601880+0000","last_change":"2024-09-13T14:15:22.549591+0000","last_active":"2024-09-13T14:15:45.601880+0000","last_peered":"2024-09-13T14:15:45.601880+0000","last_clean":"2024-09-13T14:15:45.601880+0000","last_became_active":"2024-09-13T14:15:22.549305+0000","last_became_peered":"2024-09-13T14:15:22.549305+0000","last_unstale":"2024-09-13T14:15:45.601880+0000","last_undegraded":"2024-09-13T14:15:45.601880+0000","last_fullsized":"2024-09-13T14:15:45.601880+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601943+0000","last_change":"2024-09-13T14:15:26.562976+0000","last_active":"2024-09-13T14:15:45.601943+0000","last_peered":"2024-09-13T14:15:45.601943+0000","last_clean":"2024-09-13T14:15:45.601943+0000","last_became_active":"2024-09-13T14:15:26.561391+0000","last_became_peered":"2024-09-13T14:15:26.561391+0000","last_unstale":"2024-09-13T14:15:45.601943+0000","last_undegraded":"2024-09-13T14:15:45.601943+0000","last_fullsized":"2024-09-13T14:15:45.601943+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603305+0000","last_change":"2024-09-13T14:15:22.550059+0000","last_active":"2024-09-13T14:15:45.603305+0000","last_peered":"2024-09-13T14:15:45.603305+0000","last_clean":"2024-09-13T14:15:45.603305+0000","last_became_active":"2024-09-13T14:15:22.549682+0000","last_became_peered":"2024-09-13T14:15:22.549682+0000","last_unstale":"2024-09-13T14:15:45.603305+0000","last_undegraded":"2024-09-13T14:15:45.603305+0000","last_fullsized":"2024-09-13T14:15:45.603305+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601802+0000","last_change":"2024-09-13T14:15:26.565253+0000","last_active":"2024-09-13T14:15:45.601802+0000","last_peered":"2024-09-13T14:15:45.601802+0000","last_clean":"2024-09-13T14:15:45.601802+0000","last_became_active":"2024-09-13T14:15:26.564972+0000","last_became_peered":"2024-09-13T14:15:26.564972+0000","last_unstale":"2024-09-13T14:15:45.601802+0000","last_undegraded":"2024-09-13T14:15:45.601802+0000","last_fullsized":"2024-09-13T14:15:45.601802+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602304+0000","last_change":"2024-09-13T14:15:22.547223+0000","last_active":"2024-09-13T14:15:45.602304+0000","last_peered":"2024-09-13T14:15:45.602304+0000","last_clean":"2024-09-13T14:15:45.602304+0000","last_became_active":"2024-09-13T14:15:22.546934+0000","last_became_peered":"2024-09-13T14:15:22.546934+0000","last_unstale":"2024-09-13T14:15:45.602304+0000","last_undegraded":"2024-09-13T14:15:45.602304+0000","last_fullsized":"2024-09-13T14:15:45.602304+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597836+0000","last_change":"2024-09-13T14:15:26.571004+0000","last_active":"2024-09-13T14:15:45.597836+0000","last_peered":"2024-09-13T14:15:45.597836+0000","last_clean":"2024-09-13T14:15:45.597836+0000","last_became_active":"2024-09-13T14:15:26.570753+0000","last_became_peered":"2024-09-13T14:15:26.570753+0000","last_unstale":"2024-09-13T14:15:45.597836+0000","last_undegraded":"2024-09-13T14:15:45.597836+0000","last_fullsized":"2024-09-13T14:15:45.597836+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.045057+0000","last_change":"2024-09-13T14:15:22.545342+0000","last_active":"2024-09-13T14:15:46.045057+0000","last_peered":"2024-09-13T14:15:46.045057+0000","last_clean":"2024-09-13T14:15:46.045057+0000","last_became_active":"2024-09-13T14:15:22.545067+0000","last_became_peered":"2024-09-13T14:15:22.545067+0000","last_unstale":"2024-09-13T14:15:46.045057+0000","last_undegraded":"2024-09-13T14:15:46.045057+0000","last_fullsized":"2024-09-13T14:15:46.045057+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.963528+0000","last_change":"2024-09-13T14:15:46.963528+0000","last_active":"2024-09-13T14:15:46.963528+0000","last_peered":"2024-09-13T14:15:46.963528+0000","last_clean":"2024-09-13T14:15:46.963528+0000","last_became_active":"2024-09-13T14:15:46.963216+0000","last_became_peered":"2024-09-13T14:15:46.963216+0000","last_unstale":"2024-09-13T14:15:46.963528+0000","last_undegraded":"2024-09-13T14:15:46.963528+0000","last_fullsized":"2024-09-13T14:15:46.963528+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:29.430917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602443+0000","last_change":"2024-09-13T14:15:22.550293+0000","last_active":"2024-09-13T14:15:45.602443+0000","last_peered":"2024-09-13T14:15:45.602443+0000","last_clean":"2024-09-13T14:15:45.602443+0000","last_became_active":"2024-09-13T14:15:22.549735+0000","last_became_peered":"2024-09-13T14:15:22.549735+0000","last_unstale":"2024-09-13T14:15:45.602443+0000","last_undegraded":"2024-09-13T14:15:45.602443+0000","last_fullsized":"2024-09-13T14:15:45.602443+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.598085+0000","last_change":"2024-09-13T14:15:26.556457+0000","last_active":"2024-09-13T14:15:45.598085+0000","last_peered":"2024-09-13T14:15:45.598085+0000","last_clean":"2024-09-13T14:15:45.598085+0000","last_became_active":"2024-09-13T14:15:26.555684+0000","last_became_peered":"2024-09-13T14:15:26.555684+0000","last_unstale":"2024-09-13T14:15:45.598085+0000","last_undegraded":"2024-09-13T14:15:45.598085+0000","last_fullsized":"2024-09-13T14:15:45.598085+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601621+0000","last_change":"2024-09-13T14:15:22.549793+0000","last_active":"2024-09-13T14:15:45.601621+0000","last_peered":"2024-09-13T14:15:45.601621+0000","last_clean":"2024-09-13T14:15:45.601621+0000","last_became_active":"2024-09-13T14:15:22.549291+0000","last_became_peered":"2024-09-13T14:15:22.549291+0000","last_unstale":"2024-09-13T14:15:45.601621+0000","last_undegraded":"2024-09-13T14:15:45.601621+0000","last_fullsized":"2024-09-13T14:15:45.601621+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.596926+0000","last_change":"2024-09-13T14:15:26.554955+0000","last_active":"2024-09-13T14:15:45.596926+0000","last_peered":"2024-09-13T14:15:45.596926+0000","last_clean":"2024-09-13T14:15:45.596926+0000","last_became_active":"2024-09-13T14:15:26.553125+0000","last_became_peered":"2024-09-13T14:15:26.553125+0000","last_unstale":"2024-09-13T14:15:45.596926+0000","last_undegraded":"2024-09-13T14:15:45.596926+0000","last_fullsized":"2024-09-13T14:15:45.596926+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.602869+0000","last_change":"2024-09-13T14:15:22.549798+0000","last_active":"2024-09-13T14:15:45.602869+0000","last_peered":"2024-09-13T14:15:45.602869+0000","last_clean":"2024-09-13T14:15:45.602869+0000","last_became_active":"2024-09-13T14:15:22.549508+0000","last_became_peered":"2024-09-13T14:15:22.549508+0000","last_unstale":"2024-09-13T14:15:45.602869+0000","last_undegraded":"2024-09-13T14:15:45.602869+0000","last_fullsized":"2024-09-13T14:15:45.602869+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597521+0000","last_change":"2024-09-13T14:15:26.571783+0000","last_active":"2024-09-13T14:15:45.597521+0000","last_peered":"2024-09-13T14:15:45.597521+0000","last_clean":"2024-09-13T14:15:45.597521+0000","last_became_active":"2024-09-13T14:15:26.571523+0000","last_became_peered":"2024-09-13T14:15:26.571523+0000","last_unstale":"2024-09-13T14:15:45.597521+0000","last_undegraded":"2024-09-13T14:15:45.597521+0000","last_fullsized":"2024-09-13T14:15:45.597521+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603649+0000","last_change":"2024-09-13T14:15:22.549324+0000","last_active":"2024-09-13T14:15:45.603649+0000","last_peered":"2024-09-13T14:15:45.603649+0000","last_clean":"2024-09-13T14:15:45.603649+0000","last_became_active":"2024-09-13T14:15:22.549022+0000","last_became_peered":"2024-09-13T14:15:22.549022+0000","last_unstale":"2024-09-13T14:15:45.603649+0000","last_undegraded":"2024-09-13T14:15:45.603649+0000","last_fullsized":"2024-09-13T14:15:45.603649+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.044615+0000","last_change":"2024-09-13T14:15:26.549648+0000","last_active":"2024-09-13T14:15:46.044615+0000","last_peered":"2024-09-13T14:15:46.044615+0000","last_clean":"2024-09-13T14:15:46.044615+0000","last_became_active":"2024-09-13T14:15:26.549328+0000","last_became_peered":"2024-09-13T14:15:26.549328+0000","last_unstale":"2024-09-13T14:15:46.044615+0000","last_undegraded":"2024-09-13T14:15:46.044615+0000","last_fullsized":"2024-09-13T14:15:46.044615+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.601847+0000","last_change":"2024-09-13T14:15:26.573588+0000","last_active":"2024-09-13T14:15:45.601847+0000","last_peered":"2024-09-13T14:15:45.601847+0000","last_clean":"2024-09-13T14:15:45.601847+0000","last_became_active":"2024-09-13T14:15:26.572830+0000","last_became_peered":"2024-09-13T14:15:26.572830+0000","last_unstale":"2024-09-13T14:15:45.601847+0000","last_undegraded":"2024-09-13T14:15:45.601847+0000","last_fullsized":"2024-09-13T14:15:45.601847+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603125+0000","last_change":"2024-09-13T14:15:22.550180+0000","last_active":"2024-09-13T14:15:45.603125+0000","last_peered":"2024-09-13T14:15:45.603125+0000","last_clean":"2024-09-13T14:15:45.603125+0000","last_became_active":"2024-09-13T14:15:22.549754+0000","last_became_peered":"2024-09-13T14:15:22.549754+0000","last_unstale":"2024-09-13T14:15:45.603125+0000","last_undegraded":"2024-09-13T14:15:45.603125+0000","last_fullsized":"2024-09-13T14:15:45.603125+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.963486+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:15:46.963486+0000","last_peered":"2024-09-13T14:15:46.963486+0000","last_clean":"2024-09-13T14:15:46.963486+0000","last_became_active":"2024-09-13T14:15:46.963081+0000","last_became_peered":"2024-09-13T14:15:46.963081+0000","last_unstale":"2024-09-13T14:15:46.963486+0000","last_undegraded":"2024-09-13T14:15:46.963486+0000","last_fullsized":"2024-09-13T14:15:46.963486+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:51.607549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.603096+0000","last_change":"2024-09-13T14:15:22.550265+0000","last_active":"2024-09-13T14:15:45.603096+0000","last_peered":"2024-09-13T14:15:45.603096+0000","last_clean":"2024-09-13T14:15:45.603096+0000","last_became_active":"2024-09-13T14:15:22.549179+0000","last_became_peered":"2024-09-13T14:15:22.549179+0000","last_unstale":"2024-09-13T14:15:45.603096+0000","last_undegraded":"2024-09-13T14:15:45.603096+0000","last_fullsized":"2024-09-13T14:15:45.603096+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.599902+0000","last_change":"2024-09-13T14:15:26.575732+0000","last_active":"2024-09-13T14:15:45.599902+0000","last_peered":"2024-09-13T14:15:45.599902+0000","last_clean":"2024-09-13T14:15:45.599902+0000","last_became_active":"2024-09-13T14:15:26.575235+0000","last_became_peered":"2024-09-13T14:15:26.575235+0000","last_unstale":"2024-09-13T14:15:45.599902+0000","last_undegraded":"2024-09-13T14:15:45.599902+0000","last_fullsized":"2024-09-13T14:15:45.599902+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597924+0000","last_change":"2024-09-13T14:15:22.542901+0000","last_active":"2024-09-13T14:15:45.597924+0000","last_peered":"2024-09-13T14:15:45.597924+0000","last_clean":"2024-09-13T14:15:45.597924+0000","last_became_active":"2024-09-13T14:15:22.542604+0000","last_became_peered":"2024-09-13T14:15:22.542604+0000","last_unstale":"2024-09-13T14:15:45.597924+0000","last_undegraded":"2024-09-13T14:15:45.597924+0000","last_fullsized":"2024-09-13T14:15:45.597924+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:15:45.597651+0000","last_change":"2024-09-13T14:15:26.555625+0000","last_active":"2024-09-13T14:15:45.597651+0000","last_peered":"2024-09-13T14:15:45.597651+0000","last_clean":"2024-09-13T14:15:45.597651+0000","last_became_active":"2024-09-13T14:15:26.554633+0000","last_became_peered":"2024-09-13T14:15:26.554633+0000","last_unstale":"2024-09-13T14:15:45.597651+0000","last_undegraded":"2024-09-13T14:15:45.597651+0000","last_fullsized":"2024-09-13T14:15:45.597651+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":36,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:15:46.609525+0000","last_change":"2024-09-13T14:15:46.609525+0000","last_active":"2024-09-13T14:15:46.609525+0000","last_peered":"2024-09-13T14:15:46.609525+0000","last_clean":"2024-09-13T14:15:46.609525+0000","last_became_active":"2024-09-13T14:15:46.605752+0000","last_became_peered":"2024-09-13T14:15:46.605752+0000","last_unstale":"2024-09-13T14:15:46.609525+0000","last_undegraded":"2024-09-13T14:15:46.609525+0000","last_fullsized":"2024-09-13T14:15:46.609525+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:29.004573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430222,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298520,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446632,"statfs":{"total":95995035648,"available":95689351168,"internally_reserved":0,"allocated":1187840,"data_stored":841478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46899999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]}]},{"osd":6,"up_from":42,"seq":180388626450,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298520,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446632,"statfs":{"total":95995035648,"available":95689351168,"internally_reserved":0,"allocated":1191936,"data_stored":842714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.47199999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.439}]},{"osd":2,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.58799999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40100000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.50700000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:15:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.628}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":252100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.65500000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60799999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.623},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56100000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.58899999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.45600000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:15:31 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.437},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.49299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.45600000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82999999999999996,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.61499999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56999999999999995,"15min":0.56399999999999995},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78400000000000003,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54800000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.85299999999999998,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41799999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57499999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:15:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.88200000000000001,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.56999999999999995},"min":{"1min":0.46800000000000003,"5min":0.374,"15min":0.374},"max":{"1min":0.81699999999999995,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:15:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.39200000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66900000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182433,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":252744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.53200000000000003,"5min":0.628,"15min":0.628},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.32500000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55000000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.30599999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:15:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.40500000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:15:48 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.58499999999999996,"15min":0.57299999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88900000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.629},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.59299999999999997,"15min":0.58199999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.92100000000000004,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.61399999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:15:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66300000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.53800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411365,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":593920,"data_stored":251658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.379,"15min":0.374},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.39300000000000002,"15min":0.38700000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.25800000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:15:02 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.41099999999999998,"15min":0.40600000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60199999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.40899999999999997,"15min":0.40300000000000002},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.33900000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:15:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.317},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.379}]},{"osd":4,"last update":"Fri Sep 13 14:15:44 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.59399999999999997,"15min":0.58899999999999997},"min":{"1min":0.48399999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.58899999999999997,"15min":0.58399999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.71499999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:15:07 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.625}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.85199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89400000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":250618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:15:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.30099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60999999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.501},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.40699999999999997,"15min":0.37},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.51600000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:15:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.40500000000000003,"15min":0.39000000000000001},"min":{"1min":0.28999999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.54200000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:15:25 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.42999999999999999,"15min":0.42399999999999999},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.42799999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:15:47 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55600000000000005,"15min":0.55100000000000005},"min":{"1min":0.39700000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:15:10 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.60699999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:15:35 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.70799999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1200128,"data_stored":842172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.34200000000000003},"min":{"1min":0.26100000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":0.51600000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.35799999999999998,"15min":0.34499999999999997},"min":{"1min":0.251,"5min":0.184,"15min":0.184},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.40100000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:15:01 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.29399999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.373,"15min":0.36599999999999999},"min":{"1min":0.29399999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.36699999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:15:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.34999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.438},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.61799999999999999,"5min":0.625,"15min":0.625},"last":0.41899999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:15:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.39300000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.621}]},{"osd":5,"last update":"Fri Sep 13 14:15:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57799999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:15:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:15:54.257 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:15:54.257 INFO:tasks.cephadm:Adding prometheus.a on smithi145 2024-09-13T14:15:54.257 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi145=a' 2024-09-13T14:15:54.513 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:54.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:54 smithi026 ceph-mon[27770]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 B/s, 0 objects/s recovering 2024-09-13T14:15:54.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:54 smithi026 ceph-mon[27770]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:54.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:54 smithi026 ceph-mon[39182]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 B/s, 0 objects/s recovering 2024-09-13T14:15:54.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:54 smithi026 ceph-mon[39182]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:54.928 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:54 smithi145 ceph-mon[31057]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 B/s, 0 objects/s recovering 2024-09-13T14:15:54.928 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:54 smithi145 ceph-mon[31057]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:56.351 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled prometheus update... 2024-09-13T14:15:56.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[39182]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 B/s, 0 objects/s recovering 2024-09-13T14:15:56.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:56.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[27770]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 B/s, 0 objects/s recovering 2024-09-13T14:15:56.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:56.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:56 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:56.938 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:56 smithi145 ceph-mon[31057]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 B/s, 0 objects/s recovering 2024-09-13T14:15:56.938 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:56.938 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:56 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:56.963 DEBUG:teuthology.orchestra.run.smithi145:prometheus.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T14:15:56.965 INFO:tasks.cephadm:Adding node-exporter.a on smithi026 2024-09-13T14:15:56.965 INFO:tasks.cephadm:Adding node-exporter.b on smithi145 2024-09-13T14:15:56.966 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi026=a;smithi145=b' 2024-09-13T14:15:57.257 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:57.658 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:57 smithi145 ceph-mon[31057]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:57.659 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:57 smithi145 ceph-mon[31057]: Saving service prometheus spec with placement smithi145=a;count:1 2024-09-13T14:15:57.659 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:57.659 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:57.659 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:57 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:57.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[27770]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[27770]: Saving service prometheus spec with placement smithi145=a;count:1 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[39182]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[39182]: Saving service prometheus spec with placement smithi145=a;count:1 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:57 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:15:58.674 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:58 smithi145 ceph-mon[31057]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:15:58.674 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:15:58 smithi145 ceph-mon[31057]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53 B/s, 0 objects/s recovering 2024-09-13T14:15:58.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:58 smithi026 ceph-mon[27770]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:15:58.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:15:58 smithi026 ceph-mon[27770]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53 B/s, 0 objects/s recovering 2024-09-13T14:15:58.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:58 smithi026 ceph-mon[39182]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:15:58.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:15:58 smithi026 ceph-mon[39182]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53 B/s, 0 objects/s recovering 2024-09-13T14:15:59.050 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled node-exporter update... 2024-09-13T14:16:00.218 DEBUG:teuthology.orchestra.run.smithi026:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T14:16:00.221 DEBUG:teuthology.orchestra.run.smithi145:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T14:16:00.224 INFO:tasks.cephadm:Adding alertmanager.a on smithi026 2024-09-13T14:16:00.224 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi026=a' 2024-09-13T14:16:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[27770]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[27770]: Saving service node-exporter spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[27770]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[39182]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[39182]: Saving service node-exporter spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:00 smithi026 ceph-mon[39182]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:00 smithi145 ceph-mon[31057]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi145=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:00 smithi145 ceph-mon[31057]: Saving service node-exporter spec with placement smithi026=a;smithi145=b;count:2 2024-09-13T14:16:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:00 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:00 smithi145 ceph-mon[31057]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:00.515 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:16:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:02 smithi145 ceph-mon[31057]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:02.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:02 smithi026 ceph-mon[27770]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:02 smithi026 ceph-mon[39182]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:03.399 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled alertmanager update... 2024-09-13T14:16:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:04 smithi145 ceph-mon[31057]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:04 smithi145 ceph-mon[31057]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:04 smithi145 ceph-mon[31057]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-09-13T14:16:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:04 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:04.551 DEBUG:teuthology.orchestra.run.smithi026:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T14:16:04.554 INFO:tasks.cephadm:Adding grafana.a on smithi145 2024-09-13T14:16:04.554 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi145=a' 2024-09-13T14:16:04.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[39182]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[39182]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[39182]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-09-13T14:16:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[27770]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 B/s, 0 objects/s recovering 2024-09-13T14:16:04.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[27770]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[27770]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-09-13T14:16:04.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:04 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:05.539 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:16:06.563 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:06 smithi145 ceph-mon[31057]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:06 smithi026 ceph-mon[27770]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:06.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:06 smithi026 ceph-mon[39182]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:08 smithi026 ceph-mon[27770]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:08 smithi026 ceph-mon[39182]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:08 smithi145 ceph-mon[31057]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.103 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled grafana update... 2024-09-13T14:16:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:10 smithi026 ceph-mon[27770]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:10 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:10 smithi026 ceph-mon[39182]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:10 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:10.674 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:10 smithi145 ceph-mon[31057]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.674 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:10 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:10.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:10 smithi145 systemd[1]: Starting Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:11.093 DEBUG:teuthology.orchestra.run.smithi145:grafana.a> sudo journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T14:16:11.095 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T14:16:11.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:16:11.335 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 podman[60934]: 2024-09-13 14:16:11.207791008 +0000 UTC m=+0.016758934 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:16:11.335 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 podman[60934]: 2024-09-13 14:16:11.335183071 +0000 UTC m=+0.144151001 container create 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:11.360 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:11.595 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-mon[31057]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:11.596 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-mon[31057]: Saving service grafana spec with placement smithi145=a;count:1 2024-09-13T14:16:11.596 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 podman[60934]: 2024-09-13 14:16:11.545684247 +0000 UTC m=+0.354652172 container init 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:11.596 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 podman[60934]: 2024-09-13 14:16:11.550331547 +0000 UTC m=+0.359299479 container start 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:11.596 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.574Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:16:11.596 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.574Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:16:11.596 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.574Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi145 (none))" 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.574Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.574Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.575Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.576Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.577Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.577Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.579Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.579Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.534µs 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.579Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.579Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T14:16:11.597 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.579Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.359µs wal_replay_duration=180.354µs wbl_replay_duration=122ns total_replay_duration=215.227µs 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.580Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.580Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.580Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.595Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.919582ms db_storage=664ns remote_storage=991ns web_handler=330ns query_engine=467ns scrape=369.321µs scrape_sd=21.973µs notify=729ns notify_sd=2.592µs rules=14.354054ms tracing=6.719µs 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.595Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:16:11.598 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:16:11.595Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:16:11.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:11 smithi026 ceph-mon[39182]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:11.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:11 smithi026 ceph-mon[39182]: Saving service grafana spec with placement smithi145=a;count:1 2024-09-13T14:16:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:11 smithi026 ceph-mon[27770]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:11 smithi026 ceph-mon[27770]: Saving service grafana spec with placement smithi145=a;count:1 2024-09-13T14:16:11.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 bash[60934]: 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 2024-09-13T14:16:11.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:16:11 smithi145 systemd[1]: Started Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:12.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[39182]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[27770]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:16:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mon[31057]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' 2024-09-13T14:16:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:16:12.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setuser ceph since I am not root 2024-09-13T14:16:12.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setgroup ceph since I am not root 2024-09-13T14:16:12.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-mgr[28045]: -- 172.21.15.26:0/1674074009 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559fc9f2e340 con 0x559fc9f3a800 2024-09-13T14:16:12.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:12.861+0000 7f582a851ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:16:13.099 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ignoring --setuser ceph since I am not root 2024-09-13T14:16:13.099 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ignoring --setgroup ceph since I am not root 2024-09-13T14:16:13.099 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-mgr[33131]: -- 172.21.15.145:0/187350062 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55868566e340 con 0x55868567a800 2024-09-13T14:16:13.099 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:12.857+0000 7f114ac89ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:16:13.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:12.911+0000 7f114ac89ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:13.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:12.975+0000 7f114ac89ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:16:13.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:13.098+0000 7f114ac89ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:13.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:12.921+0000 7f582a851ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:13.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:12 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:12.997+0000 7f582a851ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:16:13.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:13.128+0000 7f582a851ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:13.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:13.273+0000 7f582a851ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:16:13.370 INFO:teuthology.orchestra.run.smithi026.stdout:[client.0] 2024-09-13T14:16:13.370 INFO:teuthology.orchestra.run.smithi026.stdout: key = AQAtSeRmuVDSFRAA/WXyRj/NEUxN3PayeldpOg== 2024-09-13T14:16:13.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:13.233+0000 7f114ac89ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:16:13.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[39182]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:16:13.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[39182]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:13.556+0000 7f582a851ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:13.619+0000 7f582a851ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:13.792+0000 7f582a851ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:16:13.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[27770]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:16:13.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[27770]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-13T14:16:13.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:13.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:13 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:13.490+0000 7f114ac89ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:13.549+0000 7f114ac89ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:13.710+0000 7f114ac89ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-mon[31057]: from='mgr.14150 172.21.15.26:0/657294652' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-mon[31057]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:13 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/573606696' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:14.038 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:16:14.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T14:16:14.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T14:16:14.077 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:16:14.331 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:16:14.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:14.581+0000 7f114ac89ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:16:14.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:14.649+0000 7f114ac89ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:16:15.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:14 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:14.685+0000 7f582a851ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:16:15.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:14 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:14.752+0000 7f582a851ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:16:15.594 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.358+0000 7f114ac89ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:16:15.594 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.471+0000 7f114ac89ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:16:15.692 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:15.495+0000 7f582a851ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:16:15.692 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:15.614+0000 7f582a851ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:16:15.855 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.592+0000 7f114ac89ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:15.856 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.651+0000 7f114ac89ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:16:15.856 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.779+0000 7f114ac89ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:16:15.856 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.853+0000 7f114ac89ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:16:15.943 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:15.743+0000 7f582a851ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:15.943 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:15.806+0000 7f582a851ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:16:15.943 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:15.941+0000 7f582a851ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:16:16.118 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:15.911+0000 7f114ac89ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:16:16.119 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.048+0000 7f114ac89ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:16:16.119 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.116+0000 7f114ac89ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:16:16.218 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.013+0000 7f582a851ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:16:16.219 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.074+0000 7f582a851ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:16:16.404 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.181+0000 7f114ac89ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:16:16.404 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.265+0000 7f114ac89ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:16.404 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.344+0000 7f114ac89ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:16:16.404 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:16.402+0000 7f114ac89ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:16:16.500 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.217+0000 7f582a851ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:16:16.500 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.283+0000 7f582a851ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:16:16.500 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.352+0000 7f582a851ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:16:16.500 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.437+0000 7f582a851ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:16.546 INFO:teuthology.orchestra.run.smithi145.stdout:[client.1] 2024-09-13T14:16:16.546 INFO:teuthology.orchestra.run.smithi145.stdout: key = AQAwSeRmK/tHIBAA24GU1aliHwNw7oeo86WMVQ== 2024-09-13T14:16:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-mon[31057]: from='client.? 172.21.15.145:0/3823131355' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:16 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[27770]: from='client.? 172.21.15.145:0/3823131355' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[39182]: from='client.? 172.21.15.145:0/3823131355' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.498+0000 7f582a851ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:16.558+0000 7f582a851ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:16:17.269 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-09-13T14:16:17.269 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T14:16:17.269 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T14:16:17.307 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T14:16:17.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:17.165+0000 7f114ac89ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: [13/Sep/2024:14:16:17] ENGINE Bus STARTING 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: CherryPy Checker: 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: The Application mounted at '' has an empty config. 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: 2024-09-13T14:16:17.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: [13/Sep/2024:14:16:17] ENGINE Serving on http://:::9283 2024-09-13T14:16:17.456 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: [13/Sep/2024:14:16:17] ENGINE Bus STARTED 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: Standby manager daemon x restarted 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.345+0000 7f582a851ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:17] ENGINE Bus STARTING 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: CherryPy Checker: 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: The Application mounted at '' has an empty config. 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.468+0000 7f57d8269640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.468+0000 7f57d8269640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.468+0000 7f57d8269640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.468+0000 7f57d8269640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.468+0000 7f57d8269640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.470+0000 7f57dc271640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.470+0000 7f57dc271640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.470+0000 7f57dc271640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.470+0000 7f57dc271640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:17.470+0000 7f57dc271640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:17] ENGINE Serving on http://:::9283 2024-09-13T14:16:17.600 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:17] ENGINE Bus STARTED 2024-09-13T14:16:17.640 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:17.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: Standby manager daemon x restarted 2024-09-13T14:16:17.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: Active manager daemon y restarted 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: Activating manager daemon y 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: mgrmap e17: y(active, starting, since 0.00985403s), standbys: x 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:17.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:17.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:17.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:17.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:17.905 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: Manager daemon y is now available 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: Standby manager daemon x started 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: Active manager daemon y restarted 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: Activating manager daemon y 2024-09-13T14:16:17.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: mgrmap e17: y(active, starting, since 0.00985403s), standbys: x 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:17.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: Manager daemon y is now available 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:17.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:17.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:17 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: Standby manager daemon x restarted 2024-09-13T14:16:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/392437262' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: Active manager daemon y restarted 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: Activating manager daemon y 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: mgrmap e17: y(active, starting, since 0.00985403s), standbys: x 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:17.957 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: Manager daemon y is now available 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:17.958 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:17 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:19.070 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:18.980+0000 7f57c3600640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:16:18] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: Traceback (most recent call last): 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._connections.run(self.expiration_interval) 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._run(expiration_interval) 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:19.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: s = self.context.wrap_socket( 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: return self.sslsocket_class._create( 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self.do_handshake() 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: self._sslobj.do_handshake() 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:19.072 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:16:19.652 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:19 smithi145 ceph-mon[31057]: mgrmap e18: y(active, since 1.01274s), standbys: x 2024-09-13T14:16:19.653 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:19 smithi145 ceph-mon[31057]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:19.653 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:19 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[39182]: mgrmap e18: y(active, since 1.01274s), standbys: x 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[39182]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[27770]: mgrmap e18: y(active, since 1.01274s), standbys: x 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[27770]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:19.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:19 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.372 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'rbd' created 2024-09-13T14:16:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: [13/Sep/2024:14:16:18] ENGINE Bus STARTING 2024-09-13T14:16:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: [13/Sep/2024:14:16:18] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: [13/Sep/2024:14:16:18] ENGINE Bus STARTED 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: [13/Sep/2024:14:16:18] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2354778542' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:16:20.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: [13/Sep/2024:14:16:18] ENGINE Bus STARTING 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: [13/Sep/2024:14:16:18] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: [13/Sep/2024:14:16:18] ENGINE Bus STARTED 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: [13/Sep/2024:14:16:18] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:16:20.668 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.do_handshake() 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:20.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2354778542' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:20 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: [13/Sep/2024:14:16:18] ENGINE Bus STARTING 2024-09-13T14:16:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: [13/Sep/2024:14:16:18] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:16:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: [13/Sep/2024:14:16:18] ENGINE Bus STARTED 2024-09-13T14:16:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: [13/Sep/2024:14:16:18] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout: Traceback (most recent call last): 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._run(expiration_interval) 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: self.do_handshake() 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2354778542' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:16:20.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:20 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:20.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T14:16:21.240 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 2992M 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 2992M 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:21.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:21 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:16:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:16:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 2992M 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:21.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:21 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:22.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 2993M 2024-09-13T14:16:22.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:16:22.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:16:22.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 2993M 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:16:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:16:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:22 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 2993M 2024-09-13T14:16:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:16:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:16:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:22.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:23.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:16:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[27770]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[27770]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/196756252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:23.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[39182]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:16:23.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[39182]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:16:23.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/196756252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:23.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:23 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:23 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:23 smithi145 ceph-mon[31057]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:16:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:23 smithi145 ceph-mon[31057]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:16:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:23 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/196756252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:23 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:16:24.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[39182]: pgmap v9: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:16:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[39182]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:16:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[27770]: pgmap v9: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:16:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:24 smithi026 ceph-mon[27770]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:16:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:24 smithi145 ceph-mon[31057]: pgmap v9: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:24 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:16:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:24 smithi145 ceph-mon[31057]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:16:25.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:25 smithi026 ceph-mon[27770]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:16:25.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:25 smithi026 ceph-mon[39182]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:16:25.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:25 smithi145 ceph-mon[31057]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[39182]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[39182]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[39182]: Cluster is now healthy 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[39182]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[27770]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[27770]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[27770]: Cluster is now healthy 2024-09-13T14:16:26.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:26 smithi026 ceph-mon[27770]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:16:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:26 smithi145 ceph-mon[31057]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:16:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:26 smithi145 ceph-mon[31057]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:16:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:26 smithi145 ceph-mon[31057]: Cluster is now healthy 2024-09-13T14:16:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:26 smithi145 ceph-mon[31057]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:16:27.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T14:16:27.973 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:28.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:28 smithi026 ceph-mon[39182]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-13T14:16:28.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:28 smithi026 ceph-mon[27770]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-13T14:16:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:28 smithi145 ceph-mon[31057]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-13T14:16:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:30 smithi026 ceph-mon[27770]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 341 B/s wr, 12 op/s 2024-09-13T14:16:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:30 smithi026 ceph-mon[39182]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 341 B/s wr, 12 op/s 2024-09-13T14:16:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:30 smithi145 ceph-mon[31057]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 341 B/s wr, 12 op/s 2024-09-13T14:16:31.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:31 smithi026 podman[73210]: 2024-09-13 14:16:31.648082487 +0000 UTC m=+0.354863747 container create e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:31.975 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:16:32.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-mon[39182]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 B/s wr, 0 op/s 2024-09-13T14:16:32.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2021693496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:16:32.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-mon[27770]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 B/s wr, 0 op/s 2024-09-13T14:16:32.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2021693496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:16:32.600 INFO:teuthology.orchestra.run.smithi026.stdout:[{"version":20,"timestamp":"2024-09-13T14:16:21.003514+0000","name":"","changes":[{"name":"osd/host:smithi026/osd_memory_target","previous_value":"4212220313","new_value":"3138478489"}]}] 2024-09-13T14:16:32.600 INFO:tasks.ceph_manager:config epoch is 20 2024-09-13T14:16:32.601 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:16:32.601 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:16:32.601 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:16:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:32 smithi145 ceph-mon[31057]: pgmap v16: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 256 B/s wr, 0 op/s 2024-09-13T14:16:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:32 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2021693496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:16:32.847 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 podman[73210]: 2024-09-13 14:16:32.738699535 +0000 UTC m=+1.445480796 container init e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 podman[73210]: 2024-09-13 14:16:32.743919555 +0000 UTC m=+1.450700824 container start e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.757Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.757Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.758Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.759Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.783Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.783Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.786Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:32.786Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:16:32.848 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 bash[73210]: e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 2024-09-13T14:16:32.923 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:33.165 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:32 smithi026 systemd[1]: Started Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:33.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE Bus STOPPING 2024-09-13T14:16:33.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:16:33.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE Bus STOPPED 2024-09-13T14:16:33.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE Bus STARTING 2024-09-13T14:16:33.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:16:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE Serving on http://:::9283 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:16:33] ENGINE Bus STARTED 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:33.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:33 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:33.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:33 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:34.783 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:16:34.915 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:34.760Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000465971s 2024-09-13T14:16:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[27770]: Deploying daemon grafana.a on smithi145 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[27770]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 228 B/s wr, 0 op/s 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[27770]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[39182]: Deploying daemon grafana.a on smithi145 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[39182]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 228 B/s wr, 0 op/s 2024-09-13T14:16:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:34 smithi026 ceph-mon[39182]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:16:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:34 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:16:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:34 smithi145 ceph-mon[31057]: Deploying daemon grafana.a on smithi145 2024-09-13T14:16:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:34 smithi145 ceph-mon[31057]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 228 B/s wr, 0 op/s 2024-09-13T14:16:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:34 smithi145 ceph-mon[31057]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:16:35.595 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"flags":0,"active_gid":24425,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":295355582},{"type":"v1","addr":"172.21.15.26:6801","nonce":295355582}]},"active_addr":"172.21.15.26:6801/295355582","active_change":"2024-09-13T14:16:17.348692+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24406,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2881792973}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2245841676}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2881487463}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2209085404}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3851902768}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1460746515}]}]} 2024-09-13T14:16:35.599 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:16:35.599 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:16:35.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:16:35.868 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:35.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:35 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1536933089' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:16:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:35 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1536933089' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:16:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:35 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1536933089' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:16:36.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:36 smithi145 ceph-mon[31057]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 205 B/s wr, 0 op/s 2024-09-13T14:16:37.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:36 smithi026 ceph-mon[39182]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 205 B/s wr, 0 op/s 2024-09-13T14:16:37.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:36 smithi026 ceph-mon[27770]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 205 B/s wr, 0 op/s 2024-09-13T14:16:37.816 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:16:37.816 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":67,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:16:33.688272+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-13T14:16:19.892063+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6800/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/1842776456":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/2247322222":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/173360804":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3442380687":"2024-09-14T14:16:17.348500+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:16:38.546 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:16:38.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:16:38.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[27770]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:16:38.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:38.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2982409771' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[39182]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:16:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:38 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2982409771' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:38 smithi145 ceph-mon[31057]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:16:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:38 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2982409771' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:38.802 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:16:40.571 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":67,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:16:33.688272+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-13T14:16:19.892063+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6800/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/1842776456":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/2247322222":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/173360804":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3442380687":"2024-09-14T14:16:17.348500+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:16:40.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:40 smithi026 ceph-mon[39182]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:40.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:40 smithi026 ceph-mon[27770]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:40 smithi145 ceph-mon[31057]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:41.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:16:41.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:16:41.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:16:41.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:16:41.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:16:41.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:16:41.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:16:41.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:16:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:41 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4214578886' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:41 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/4214578886' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:41 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/4214578886' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:16:41.906 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.909 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.909 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.916 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.919 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:41.937 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:42.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:42 smithi026 ceph-mon[27770]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:42.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:42 smithi026 ceph-mon[39182]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:42 smithi145 ceph-mon[31057]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:42.915 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:16:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:16:42.762Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003252735s 2024-09-13T14:16:43.192 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:16:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:44 smithi145 ceph-mon[31057]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:44.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:44 smithi026 ceph-mon[39182]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:44.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:44 smithi026 ceph-mon[27770]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:45.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:44 smithi145 podman[63055]: 2024-09-13 14:16:44.861935254 +0000 UTC m=+0.014694875 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:16:45.604 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 podman[63055]: 2024-09-13 14:16:45.335049705 +0000 UTC m=+0.487809354 container create b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, version=8.5, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, description=Ceph Grafana Container, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-13T14:16:45.857 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 podman[63055]: 2024-09-13 14:16:45.603968444 +0000 UTC m=+0.756728052 container init b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, name=ubi8, vcs-type=git, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public) 2024-09-13T14:16:45.857 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 podman[63055]: 2024-09-13 14:16:45.609897933 +0000 UTC m=+0.762657541 container start b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, distribution-scope=public, description=Ceph Grafana Container, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, name=ubi8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc.) 2024-09-13T14:16:45.857 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 bash[63055]: b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 2024-09-13T14:16:45.857 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 systemd[1]: Started Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:45.857 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768075529Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768259757Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768279275Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768286906Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768292667Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768299584Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768305268Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768311127Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768316907Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768322555Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768327875Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768333217Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768338784Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:16:45.858 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=settings t=2024-09-13T14:16:45.768344501Z level=info msg="App mode production" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore t=2024-09-13T14:16:45.768395322Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore t=2024-09-13T14:16:45.768412871Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.769249365Z level=info msg="Starting DB migrations" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.769323084Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.769878925Z level=info msg="Executing migration" id="create user table" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.770334731Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.770798685Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.771298192Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T14:16:45.859 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.771949738Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77233701Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77300869Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.773410584Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.773769256Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.774220063Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.774582509Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77492179Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T14:16:45.860 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.775350701Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.775627659Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77601372Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77638232Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.776773432Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.777189186Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.777656618Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.779031196Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.77951566Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.779951397Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.780351234Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.781225752Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.781630663Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T14:16:45.861 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.782445886Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.782831238Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.783318253Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.783711936Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.784606908Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.785454708Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.786432195Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.78713779Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.78994552Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.79035242Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.793063234Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.795745268Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.798508112Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T14:16:45.862 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.798850137Z level=info msg="Executing migration" id="create star table" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.801574775Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.804209844Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.804636898Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.807541387Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.810252477Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.810649979Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.813284026Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.813694334Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.816636923Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.817085865Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.817390214Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.818270754Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T14:16:45.863 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.819189495Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.82004848Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.820673302Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.821554655Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.822524857Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.82353708Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.823923086Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.82478119Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.825744799Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.826469905Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.826839869Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.827166782Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.829681986Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T14:16:45.864 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.832439529Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.835131902Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.835714698Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.838760099Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.841482531Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.844327153Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.844838308Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.847598127Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.850417378Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.85314356Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.853931138Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T14:16:45.865 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.856707391Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T14:16:46.107 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.859642974Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T14:16:46.107 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.862529394Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.865375128Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.868030692Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.870962317Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.873636783Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.874570097Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.877107153Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.879690456Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.882496873Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.882845552Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.88555899Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T14:16:46.108 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.888170521Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.890864593Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.891217616Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.893963621Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.894491864Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.896621327Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.897092485Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.899421332Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.899903911Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.900892327Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.901608443Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.902642725Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.903082008Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T14:16:46.109 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.904082863Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.904762552Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.905741382Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.90651501Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.90735654Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.907854896Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.908195591Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.909039706Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.909364862Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.910231708Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.911006118Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.911357687Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T14:16:46.110 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.912197334Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T14:16:46.111 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.912638132Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.913068444Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.913908665Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.914784006Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.915600175Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.916000482Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.917013051Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.917769669Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.918815724Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.91978561Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.92066245Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.921383573Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.922408232Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T14:16:46.112 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.923072507Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.924061078Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.924760951Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.925761417Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.928458404Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.931204779Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.933999698Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.937036482Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.939579137Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.942150812Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.944983787Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.94784116Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.950470351Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T14:16:46.113 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.952995376Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.95602452Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.95877146Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.961522203Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.96439929Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.967084953Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.970104079Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.972738437Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.975370305Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.977887862Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.980743903Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.983477175Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.986060206Z level=info msg="Executing migration" id="create session table" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.988730582Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T14:16:46.114 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.9914659Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.994186514Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.996948749Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:45.999491644Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.002176807Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.004807233Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.007387607Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.010182951Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.012801386Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.015581055Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.018343362Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.021067108Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.021834555Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.022698532Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.023555915Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T14:16:46.115 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.024536174Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.025202842Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.026137097Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.026833137Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.027837393Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.028479675Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.029362056Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.030096592Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.031111191Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.032659071Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.033478565Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.03439903Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.035123723Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.03606905Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.03659292Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T14:16:46.116 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.039307326Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.039981817Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.040637563Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.041284546Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.041691591Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.041964851Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.042264591Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.042638881Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.043072791Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.043457912Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.046180612Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.04666086Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.048869714Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.051574716Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T14:16:46.117 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.054167497Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.056830986Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.059531168Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.062204815Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.064884917Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.067618702Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.070149885Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.072941617Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.075394233Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.078087689Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.080747811Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.083443695Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.086130059Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.088637709Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.091095089Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T14:16:46.118 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.091908049Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.092708278Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.09352033Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.094336539Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.095151253Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.096107137Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.097873348Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.098824749Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.099524061Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.100309473Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.100707445Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.101129737Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.101830154Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.10255449Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T14:16:46.119 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.103490235Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.104182984Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.105112435Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.106004073Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.107041459Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.107703222Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.108528497Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.109372Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T14:16:46.120 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.110228858Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T14:16:46.360 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.111083404Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T14:16:46.360 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.111946327Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T14:16:46.360 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.112785361Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.113636709Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.114436009Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.115187633Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.115582135Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.116436383Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.117301311Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.11817589Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.119160571Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.119943837Z level=info msg="Executing migration" id="create team table" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.120884723Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.123600372Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T14:16:46.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.126258675Z level=info msg="Executing migration" id="create team member table" 2024-09-13T14:16:46.362 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.129032908Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T14:16:46.362 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.131809335Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T14:16:46.362 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.134420805Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T14:16:46.362 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.137150223Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T14:16:46.362 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.139762657Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.142586196Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.145055409Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.147996124Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.150665976Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.153226121Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.1538714Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.156732207Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.159547936Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.160159449Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.16059504Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.160967116Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T14:16:46.363 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.161294301Z level=info msg="Executing migration" id="create tag table" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.16167377Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.162101562Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.162515622Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.162956443Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.163366916Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.165230038Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.165644877Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.16609458Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.166438331Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T14:16:46.364 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.166776064Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.167170048Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.167628971Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.167900671Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.168669426Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.169452012Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.170239668Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.171038976Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.171487153Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.172277156Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.172697578Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.173161484Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T14:16:46.365 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.173611618Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.176330564Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.17679691Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.177218537Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.177969241Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.178402448Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.178854404Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.179285826Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.179726441Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.17999996Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.180334415Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.18078615Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.183458707Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T14:16:46.366 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.183950459Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.186510434Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.186956667Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.189686695Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.190145832Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.192825991Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.195673191Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.198104821Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T14:16:46.367 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.198423587Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T14:16:46.368 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.200960722Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T14:16:46.368 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.201409502Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T14:16:46.368 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.203669536Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T14:16:46.368 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.204130122Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T14:16:46.368 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.2062095Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.209112109Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.211589492Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.214310742Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.216824956Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.217265692Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.218202518Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.22170874Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.225036478Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.225860743Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.226701444Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.227580165Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.228432657Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T14:16:46.369 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.228887401Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.229762531Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.230703812Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.231436742Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.232303447Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.233140841Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.234016532Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.23485316Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.235310921Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.236294486Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.237196199Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.238026404Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.238873264Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T14:16:46.370 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.241535912Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.244169982Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.247045743Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.24750347Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.250297345Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.253070912Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.255989318Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.256841062Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.259713595Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.262384764Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.26487294Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.265199817Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.268060886Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T14:16:46.371 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.270974027Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.273523404Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.274460148Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.277342159Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.280307381Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.282783319Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.28324714Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.286049539Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.288843806Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.291488264Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.292104211Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.294838376Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.297623384Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T14:16:46.372 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.300204338Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.300723126Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.30081248Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.303481612Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.306057357Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.308842483Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.309098428Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.309213765Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.309611021Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.312320645Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.315193028Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.317776959Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.318377449Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T14:16:46.373 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.319176162Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.320120021Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.320837355Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.321273195Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.322075212Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.326329396Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.327253109Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.327576608Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.331822997Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.336025689Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.337011689Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.337473262Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T14:16:46.374 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.338361119Z level=info msg="Executing migration" id="create permission table" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.338857488Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.339286246Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.34028015Z level=info msg="Executing migration" id="create role table" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.340721231Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.341654632Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.342578729Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.343431718Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.343873649Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.344855174Z level=info msg="Executing migration" id="create team role table" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.345580368Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.346582022Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.347170865Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T14:16:46.375 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.348041523Z level=info msg="Executing migration" id="create user role table" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.348826506Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.349706701Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.350164503Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.351199745Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.351886011Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.352740951Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.35319238Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.354181555Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.357022084Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.35974623Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.362520352Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T14:16:46.376 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.365412157Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[27770]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[39182]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-mon[31057]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.36804458Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.370607991Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.371577677Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.37435017Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.37706281Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.379750144Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.380171768Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.383045666Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.385965849Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.388715253Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T14:16:46.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.389110143Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.39216077Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.394882328Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.397559526Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.39788231Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.400731489Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.403378296Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.406126573Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.407150377Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T14:16:46.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.410064103Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.412803245Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.412987154Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.41542885Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.415535969Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.415823864Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.418531751Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.421380241Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.423976182Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.42444846Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.427115813Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:16:46.708 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.427560433Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.429949734Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.432455594Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.433135925Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.433888787Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.434347956Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.437470352Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.438533843Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.439560992Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.440511081Z level=info msg="Executing migration" id="add share column" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.441595183Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.44247584Z level=info msg="Executing migration" id="create file table" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.442943358Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.443764102Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T14:16:46.709 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.444478589Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.444893296Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.445356245Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.446161312Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.446577378Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.447030445Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.447516782Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.448591673Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.448916155Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.449802637Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.450710864Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T14:16:46.710 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.451456033Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.451857856Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.452850524Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.453879006Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.454750376Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.46454677Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.465547824Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.466241823Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T14:16:46.711 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.469641092Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.469975115Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.472779043Z level=info msg="Executing migration" id="create folder table" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.475390585Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.478109301Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.480951426Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.483661583Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=migrator t=2024-09-13T14:16:46.486199427Z level=info msg="migrations completed" performed=485 skipped=0 duration=716.896401ms 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore t=2024-09-13T14:16:46.486615739Z level=info msg="Created default organization" 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=plugin.loader t=2024-09-13T14:16:46.545034684Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=plugin.loader t=2024-09-13T14:16:46.576023011Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:16:46.712 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=plugin.loader t=2024-09-13T14:16:46.576050669Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:16:46.713 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=secrets t=2024-09-13T14:16:46.576092448Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:16:46.713 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=query_data t=2024-09-13T14:16:46.57795495Z level=info msg="Query Service initialization" 2024-09-13T14:16:46.713 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=live.push_http t=2024-09-13T14:16:46.580180567Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:16:47.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[27770]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:16:47.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:47.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:47.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[39182]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:16:47.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:47.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:47 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-mon[31057]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:16:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=infra.usagestats.collector t=2024-09-13T14:16:47.289624243Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=provisioning.datasources t=2024-09-13T14:16:47.289947191Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=provisioning.datasources t=2024-09-13T14:16:47.322529992Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=provisioning.datasources t=2024-09-13T14:16:47.353968798Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=provisioning.alerting t=2024-09-13T14:16:47.375376594Z level=info msg="starting to provision alerting" 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=provisioning.alerting t=2024-09-13T14:16:47.375413446Z level=info msg="finished to provision alerting" 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=ngalert.state.manager t=2024-09-13T14:16:47.375476234Z level=info msg="Warming state cache for startup" 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=grafanaStorageLogger t=2024-09-13T14:16:47.375588956Z level=info msg="storage starting" 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=http.server t=2024-09-13T14:16:47.380981349Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=ngalert.state.manager t=2024-09-13T14:16:47.385831574Z level=info msg="State cache has been initialized" states=0 duration=10.354629ms 2024-09-13T14:16:47.706 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=ticker t=2024-09-13T14:16:47.385891159Z level=info msg=starting first_tick=2024-09-13T14:16:50Z 2024-09-13T14:16:47.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:16:47.3859027Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:16:47.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore.transactions t=2024-09-13T14:16:47.386778269Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:16:47.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore.transactions t=2024-09-13T14:16:47.421242736Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:16:47.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore.transactions t=2024-09-13T14:16:47.462124724Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:16:47.707 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:16:47 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[63070]: logger=sqlstore.transactions t=2024-09-13T14:16:47.472700613Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-13T14:16:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:48 smithi145 ceph-mon[31057]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:48.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:48 smithi026 ceph-mon[39182]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:48 smithi026 ceph-mon[27770]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:49.916 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:49 smithi026 systemd[1]: Starting Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:50 smithi026 ceph-mon[27770]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:50.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:50 smithi026 ceph-mon[39182]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:50 smithi145 ceph-mon[31057]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.259 INFO:teuthology.orchestra.run.smithi026.stdout:34359738426 2024-09-13T14:16:52.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:16:52.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:52 smithi026 ceph-mon[39182]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:52 smithi026 ceph-mon[27770]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:52 smithi145 ceph-mon[31057]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:53.204 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:53 smithi026 bash[75387]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:16:53.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:16:52 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:16:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:53.528 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:53 smithi026 bash[75387]: Getting image source signatures 2024-09-13T14:16:53.528 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:53 smithi026 bash[75387]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:16:53.528 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:53 smithi026 bash[75387]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:16:53.528 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:53 smithi026 bash[75387]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:16:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:54 smithi145 ceph-mon[31057]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:54.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:54 smithi026 ceph-mon[27770]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:54.788 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:54 smithi026 ceph-mon[39182]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:55.555 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:55 smithi026 bash[75387]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:16:55.915 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:55 smithi026 bash[75387]: Writing manifest to image destination 2024-09-13T14:16:56.396 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 podman[75387]: 2024-09-13 14:16:56.104739067 +0000 UTC m=+3.613912337 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:16:56.396 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 podman[75387]: 2024-09-13 14:16:56.194870412 +0000 UTC m=+3.704043683 container create 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-mon[27770]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-mon[39182]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:56 smithi145 ceph-mon[31057]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:57.268 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 podman[75387]: 2024-09-13 14:16:56.932642539 +0000 UTC m=+4.441815811 container init 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:57.268 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 podman[75387]: 2024-09-13 14:16:56.938800888 +0000 UTC m=+4.447974166 container start 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:57.268 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.943Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:16:57.268 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.943Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.944Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.944Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.944Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.944Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:16:57.269 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:16:57.270 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:16:57.271 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:16:57.272 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[75701]: ts=2024-09-13T14:16:56.945Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:57 smithi026 bash[75387]: 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 2024-09-13T14:16:57.273 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:16:57 smithi026 systemd[1]: Started Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:58.054 INFO:teuthology.orchestra.run.smithi026.stdout:206158430235 2024-09-13T14:16:58.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:16:58.115 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:16:58 smithi145 bash[63336]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[27770]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[27770]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[39182]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:16:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:16:58 smithi026 ceph-mon[39182]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:16:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:58 smithi145 ceph-mon[31057]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:16:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:16:58 smithi145 ceph-mon[31057]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:58.542 INFO:teuthology.orchestra.run.smithi026.stdout:180388626463 2024-09-13T14:16:58.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:16:59.190 INFO:teuthology.orchestra.run.smithi026.stdout:55834574903 2024-09-13T14:16:59.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:16:59.978 INFO:teuthology.orchestra.run.smithi026.stdout:128849018920 2024-09-13T14:16:59.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:17:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:00 smithi026 ceph-mon[27770]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:00 smithi026 ceph-mon[39182]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:00 smithi145 ceph-mon[31057]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:00.706 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:00 smithi145 bash[63336]: Getting image source signatures 2024-09-13T14:17:00.706 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:00 smithi145 bash[63336]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:17:00.706 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:00 smithi145 bash[63336]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:17:00.706 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:00 smithi145 bash[63336]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:17:00.742 INFO:teuthology.orchestra.run.smithi026.stdout:77309411378 2024-09-13T14:17:00.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:17:01.129 INFO:teuthology.orchestra.run.smithi026.stdout:154618822691 2024-09-13T14:17:01.129 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:17:01.375 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:01.978 INFO:teuthology.orchestra.run.smithi026.stdout:107374182447 2024-09-13T14:17:01.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:17:01.979 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:01 smithi145 bash[63336]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:17:02.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:01 smithi145 bash[63336]: Writing manifest to image destination 2024-09-13T14:17:02.516 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-mon[31057]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:02.516 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:02.517 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 podman[63336]: 2024-09-13 14:17:02.182117442 +0000 UTC m=+4.266315753 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:17:02.517 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 podman[63336]: 2024-09-13 14:17:02.297611385 +0000 UTC m=+4.381809696 container create 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:02 smithi026 ceph-mon[39182]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:02.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:02 smithi026 ceph-mon[27770]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:02.669 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.669 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.669 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.671 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.724 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:02.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:17:02.955 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 podman[63336]: 2024-09-13 14:17:02.516505756 +0000 UTC m=+4.600704067 container init 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:02.955 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 podman[63336]: 2024-09-13 14:17:02.521557213 +0000 UTC m=+4.605755524 container start 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:02.955 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:17:02.955 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.525Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:17:02.956 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:17:02.957 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:17:02.958 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:17:02.959 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.526Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:17:02.960 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[63411]: ts=2024-09-13T14:17:02.527Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:17:02.961 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 bash[63336]: 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 2024-09-13T14:17:02.961 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:17:02 smithi145 systemd[1]: Started Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:03.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:03.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:04.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:04 smithi145 ceph-mon[31057]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:04.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:04 smithi026 ceph-mon[27770]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:04.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:04 smithi026 ceph-mon[39182]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:06.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[27770]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:06.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:06.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[39182]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:06 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:06.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:06 smithi145 ceph-mon[31057]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:06.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:06 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:06.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:06 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:07.782 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:08.416 INFO:teuthology.orchestra.run.smithi026.stdout:154618822693 2024-09-13T14:17:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[27770]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/488430471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:17:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[39182]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:08 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/488430471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:17:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:08 smithi145 ceph-mon[31057]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:08 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:08 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/488430471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:17:09.006 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822691 got 154618822693 for osd.5 2024-09-13T14:17:09.006 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:10 smithi026 ceph-mon[27770]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:10 smithi026 ceph-mon[39182]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:10 smithi145 ceph-mon[31057]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:10.955 INFO:teuthology.orchestra.run.smithi026.stdout:55834574906 2024-09-13T14:17:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:11 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3866566429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:17:11.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:11 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3866566429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:17:11.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:11 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3866566429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:17:12.190 INFO:teuthology.orchestra.run.smithi026.stdout:128849018922 2024-09-13T14:17:12.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:12 smithi026 ceph-mon[39182]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:12.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:12 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/503717533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:17:12.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:12 smithi026 ceph-mon[27770]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:12.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:12 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/503717533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:17:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:12 smithi145 ceph-mon[31057]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:12 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/503717533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:17:13.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:17:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:17:13.511 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574903 got 55834574906 for osd.1 2024-09-13T14:17:13.511 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:13.635 INFO:teuthology.orchestra.run.smithi026.stdout:34359738432 2024-09-13T14:17:13.992 INFO:teuthology.orchestra.run.smithi026.stdout:206158430239 2024-09-13T14:17:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:14 smithi145 ceph-mon[31057]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:14 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/4012085450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:17:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:14 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/463694741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[39182]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/4012085450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/463694741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[27770]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4012085450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:17:14.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:14 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/463694741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:17:14.839 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018920 got 128849018922 for osd.4 2024-09-13T14:17:14.840 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:15.465 INFO:teuthology.orchestra.run.smithi026.stdout:77309411382 2024-09-13T14:17:15.707 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430235 got 206158430239 for osd.7 2024-09-13T14:17:15.707 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:16 smithi026 ceph-mon[27770]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:16 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3394253826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:17:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:16 smithi026 ceph-mon[39182]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:16 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3394253826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:17:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:16 smithi145 ceph-mon[31057]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:16 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3394253826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:17:16.827 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738426 got 34359738432 for osd.0 2024-09-13T14:17:16.827 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:17.549 INFO:teuthology.orchestra.run.smithi026.stdout:180388626467 2024-09-13T14:17:17.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:17.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:17.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:17.795 INFO:teuthology.orchestra.run.smithi026.stdout:107374182450 2024-09-13T14:17:18.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626463 got 180388626467 for osd.6 2024-09-13T14:17:18.311 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:18.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[27770]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:18.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/76688304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:17:18.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4132862098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:17:18.658 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[39182]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:18.658 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/76688304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:17:18.658 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:18 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/4132862098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:17:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:18 smithi145 ceph-mon[31057]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:18 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/76688304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:17:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:18 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/4132862098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:17:18.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182447 got 107374182450 for osd.3 2024-09-13T14:17:18.963 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:19.475 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411378 got 77309411382 for osd.2 2024-09-13T14:17:19.475 DEBUG:teuthology.parallel:result is None 2024-09-13T14:17:19.475 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:17:19.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:17:20.334 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:20 smithi145 ceph-mon[31057]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:20.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:20 smithi026 ceph-mon[39182]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:20.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:20 smithi026 ceph-mon[27770]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.387 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:22.389 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-09-13T14:17:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:22 smithi026 ceph-mon[27770]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:22 smithi026 ceph-mon[39182]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:22 smithi145 ceph-mon[31057]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:23.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:17:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:17:23.249 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-09-13T14:17:21.375573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":590938,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386048,"kb_used_data":7204,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575168,"statfs":{"total":767960285184,"available":765516972032,"internally_reserved":0,"allocated":7376896,"data_stored":4258759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002967"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427311+0000","last_change":"2024-09-13T14:15:26.568393+0000","last_active":"2024-09-13T14:16:25.427311+0000","last_peered":"2024-09-13T14:16:25.427311+0000","last_clean":"2024-09-13T14:16:25.427311+0000","last_became_active":"2024-09-13T14:15:26.568075+0000","last_became_peered":"2024-09-13T14:15:26.568075+0000","last_unstale":"2024-09-13T14:16:25.427311+0000","last_undegraded":"2024-09-13T14:16:25.427311+0000","last_fullsized":"2024-09-13T14:16:25.427311+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403288+0000","last_change":"2024-09-13T14:15:26.563624+0000","last_active":"2024-09-13T14:16:25.403288+0000","last_peered":"2024-09-13T14:16:25.403288+0000","last_clean":"2024-09-13T14:16:25.403288+0000","last_became_active":"2024-09-13T14:15:26.563364+0000","last_became_peered":"2024-09-13T14:15:26.563364+0000","last_unstale":"2024-09-13T14:16:25.403288+0000","last_undegraded":"2024-09-13T14:16:25.403288+0000","last_fullsized":"2024-09-13T14:16:25.403288+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426637+0000","last_change":"2024-09-13T14:15:26.566586+0000","last_active":"2024-09-13T14:16:25.426637+0000","last_peered":"2024-09-13T14:16:25.426637+0000","last_clean":"2024-09-13T14:16:25.426637+0000","last_became_active":"2024-09-13T14:15:26.566331+0000","last_became_peered":"2024-09-13T14:15:26.566331+0000","last_unstale":"2024-09-13T14:16:25.426637+0000","last_undegraded":"2024-09-13T14:16:25.426637+0000","last_fullsized":"2024-09-13T14:16:25.426637+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404976+0000","last_change":"2024-09-13T14:15:26.564769+0000","last_active":"2024-09-13T14:16:25.404976+0000","last_peered":"2024-09-13T14:16:25.404976+0000","last_clean":"2024-09-13T14:16:25.404976+0000","last_became_active":"2024-09-13T14:15:26.564496+0000","last_became_peered":"2024-09-13T14:15:26.564496+0000","last_unstale":"2024-09-13T14:16:25.404976+0000","last_undegraded":"2024-09-13T14:16:25.404976+0000","last_fullsized":"2024-09-13T14:16:25.404976+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427347+0000","last_change":"2024-09-13T14:15:26.555592+0000","last_active":"2024-09-13T14:16:25.427347+0000","last_peered":"2024-09-13T14:16:25.427347+0000","last_clean":"2024-09-13T14:16:25.427347+0000","last_became_active":"2024-09-13T14:15:26.555311+0000","last_became_peered":"2024-09-13T14:15:26.555311+0000","last_unstale":"2024-09-13T14:16:25.427347+0000","last_undegraded":"2024-09-13T14:16:25.427347+0000","last_fullsized":"2024-09-13T14:16:25.427347+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404742+0000","last_change":"2024-09-13T14:15:26.572204+0000","last_active":"2024-09-13T14:16:25.404742+0000","last_peered":"2024-09-13T14:16:25.404742+0000","last_clean":"2024-09-13T14:16:25.404742+0000","last_became_active":"2024-09-13T14:15:26.571310+0000","last_became_peered":"2024-09-13T14:15:26.571310+0000","last_unstale":"2024-09-13T14:16:25.404742+0000","last_undegraded":"2024-09-13T14:16:25.404742+0000","last_fullsized":"2024-09-13T14:16:25.404742+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426113+0000","last_change":"2024-09-13T14:15:26.566676+0000","last_active":"2024-09-13T14:16:25.426113+0000","last_peered":"2024-09-13T14:16:25.426113+0000","last_clean":"2024-09-13T14:16:25.426113+0000","last_became_active":"2024-09-13T14:15:26.566403+0000","last_became_peered":"2024-09-13T14:15:26.566403+0000","last_unstale":"2024-09-13T14:16:25.426113+0000","last_undegraded":"2024-09-13T14:16:25.426113+0000","last_fullsized":"2024-09-13T14:16:25.426113+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399106+0000","last_change":"2024-09-13T14:15:26.566663+0000","last_active":"2024-09-13T14:16:25.399106+0000","last_peered":"2024-09-13T14:16:25.399106+0000","last_clean":"2024-09-13T14:16:25.399106+0000","last_became_active":"2024-09-13T14:15:26.566400+0000","last_became_peered":"2024-09-13T14:15:26.566400+0000","last_unstale":"2024-09-13T14:16:25.399106+0000","last_undegraded":"2024-09-13T14:16:25.399106+0000","last_fullsized":"2024-09-13T14:16:25.399106+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427605+0000","last_change":"2024-09-13T14:15:26.570338+0000","last_active":"2024-09-13T14:16:25.427605+0000","last_peered":"2024-09-13T14:16:25.427605+0000","last_clean":"2024-09-13T14:16:25.427605+0000","last_became_active":"2024-09-13T14:15:26.569776+0000","last_became_peered":"2024-09-13T14:15:26.569776+0000","last_unstale":"2024-09-13T14:16:25.427605+0000","last_undegraded":"2024-09-13T14:16:25.427605+0000","last_fullsized":"2024-09-13T14:16:25.427605+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427114+0000","last_change":"2024-09-13T14:15:26.564796+0000","last_active":"2024-09-13T14:16:25.427114+0000","last_peered":"2024-09-13T14:16:25.427114+0000","last_clean":"2024-09-13T14:16:25.427114+0000","last_became_active":"2024-09-13T14:15:26.563959+0000","last_became_peered":"2024-09-13T14:15:26.563959+0000","last_unstale":"2024-09-13T14:16:25.427114+0000","last_undegraded":"2024-09-13T14:16:25.427114+0000","last_fullsized":"2024-09-13T14:16:25.427114+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426507+0000","last_change":"2024-09-13T14:15:26.567040+0000","last_active":"2024-09-13T14:16:25.426507+0000","last_peered":"2024-09-13T14:16:25.426507+0000","last_clean":"2024-09-13T14:16:25.426507+0000","last_became_active":"2024-09-13T14:15:26.566726+0000","last_became_peered":"2024-09-13T14:15:26.566726+0000","last_unstale":"2024-09-13T14:16:25.426507+0000","last_undegraded":"2024-09-13T14:16:25.426507+0000","last_fullsized":"2024-09-13T14:16:25.426507+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398688+0000","last_change":"2024-09-13T14:15:26.563679+0000","last_active":"2024-09-13T14:16:25.398688+0000","last_peered":"2024-09-13T14:16:25.398688+0000","last_clean":"2024-09-13T14:16:25.398688+0000","last_became_active":"2024-09-13T14:15:26.563402+0000","last_became_peered":"2024-09-13T14:15:26.563402+0000","last_unstale":"2024-09-13T14:16:25.398688+0000","last_undegraded":"2024-09-13T14:16:25.398688+0000","last_fullsized":"2024-09-13T14:16:25.398688+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399996+0000","last_change":"2024-09-13T14:15:26.552634+0000","last_active":"2024-09-13T14:16:25.399996+0000","last_peered":"2024-09-13T14:16:25.399996+0000","last_clean":"2024-09-13T14:16:25.399996+0000","last_became_active":"2024-09-13T14:15:26.552354+0000","last_became_peered":"2024-09-13T14:15:26.552354+0000","last_unstale":"2024-09-13T14:16:25.399996+0000","last_undegraded":"2024-09-13T14:16:25.399996+0000","last_fullsized":"2024-09-13T14:16:25.399996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426019+0000","last_change":"2024-09-13T14:15:26.576136+0000","last_active":"2024-09-13T14:16:25.426019+0000","last_peered":"2024-09-13T14:16:25.426019+0000","last_clean":"2024-09-13T14:16:25.426019+0000","last_became_active":"2024-09-13T14:15:26.575737+0000","last_became_peered":"2024-09-13T14:15:26.575737+0000","last_unstale":"2024-09-13T14:16:25.426019+0000","last_undegraded":"2024-09-13T14:16:25.426019+0000","last_fullsized":"2024-09-13T14:16:25.426019+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402570+0000","last_change":"2024-09-13T14:15:26.566237+0000","last_active":"2024-09-13T14:16:25.402570+0000","last_peered":"2024-09-13T14:16:25.402570+0000","last_clean":"2024-09-13T14:16:25.402570+0000","last_became_active":"2024-09-13T14:15:26.565645+0000","last_became_peered":"2024-09-13T14:15:26.565645+0000","last_unstale":"2024-09-13T14:16:25.402570+0000","last_undegraded":"2024-09-13T14:16:25.402570+0000","last_fullsized":"2024-09-13T14:16:25.402570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404637+0000","last_change":"2024-09-13T14:15:46.965107+0000","last_active":"2024-09-13T14:16:25.404637+0000","last_peered":"2024-09-13T14:16:25.404637+0000","last_clean":"2024-09-13T14:16:25.404637+0000","last_became_active":"2024-09-13T14:15:46.964708+0000","last_became_peered":"2024-09-13T14:15:46.964708+0000","last_unstale":"2024-09-13T14:16:25.404637+0000","last_undegraded":"2024-09-13T14:16:25.404637+0000","last_fullsized":"2024-09-13T14:16:25.404637+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405223+0000","last_change":"2024-09-13T14:15:26.565535+0000","last_active":"2024-09-13T14:16:25.405223+0000","last_peered":"2024-09-13T14:16:25.405223+0000","last_clean":"2024-09-13T14:16:25.405223+0000","last_became_active":"2024-09-13T14:15:26.565091+0000","last_became_peered":"2024-09-13T14:15:26.565091+0000","last_unstale":"2024-09-13T14:16:25.405223+0000","last_undegraded":"2024-09-13T14:16:25.405223+0000","last_fullsized":"2024-09-13T14:16:25.405223+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398039+0000","last_change":"2024-09-13T14:15:26.556686+0000","last_active":"2024-09-13T14:16:25.398039+0000","last_peered":"2024-09-13T14:16:25.398039+0000","last_clean":"2024-09-13T14:16:25.398039+0000","last_became_active":"2024-09-13T14:15:26.556349+0000","last_became_peered":"2024-09-13T14:15:26.556349+0000","last_unstale":"2024-09-13T14:16:25.398039+0000","last_undegraded":"2024-09-13T14:16:25.398039+0000","last_fullsized":"2024-09-13T14:16:25.398039+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427100+0000","last_change":"2024-09-13T14:15:26.564825+0000","last_active":"2024-09-13T14:16:25.427100+0000","last_peered":"2024-09-13T14:16:25.427100+0000","last_clean":"2024-09-13T14:16:25.427100+0000","last_became_active":"2024-09-13T14:15:26.563703+0000","last_became_peered":"2024-09-13T14:15:26.563703+0000","last_unstale":"2024-09-13T14:16:25.427100+0000","last_undegraded":"2024-09-13T14:16:25.427100+0000","last_fullsized":"2024-09-13T14:16:25.427100+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426232+0000","last_change":"2024-09-13T14:15:26.568327+0000","last_active":"2024-09-13T14:16:25.426232+0000","last_peered":"2024-09-13T14:16:25.426232+0000","last_clean":"2024-09-13T14:16:25.426232+0000","last_became_active":"2024-09-13T14:15:26.568070+0000","last_became_peered":"2024-09-13T14:15:26.568070+0000","last_unstale":"2024-09-13T14:16:25.426232+0000","last_undegraded":"2024-09-13T14:16:25.426232+0000","last_fullsized":"2024-09-13T14:16:25.426232+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426139+0000","last_change":"2024-09-13T14:15:26.568339+0000","last_active":"2024-09-13T14:16:25.426139+0000","last_peered":"2024-09-13T14:16:25.426139+0000","last_clean":"2024-09-13T14:16:25.426139+0000","last_became_active":"2024-09-13T14:15:26.568056+0000","last_became_peered":"2024-09-13T14:15:26.568056+0000","last_unstale":"2024-09-13T14:16:25.426139+0000","last_undegraded":"2024-09-13T14:16:25.426139+0000","last_fullsized":"2024-09-13T14:16:25.426139+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404854+0000","last_change":"2024-09-13T14:15:26.572417+0000","last_active":"2024-09-13T14:16:25.404854+0000","last_peered":"2024-09-13T14:16:25.404854+0000","last_clean":"2024-09-13T14:16:25.404854+0000","last_became_active":"2024-09-13T14:15:26.572126+0000","last_became_peered":"2024-09-13T14:15:26.572126+0000","last_unstale":"2024-09-13T14:16:25.404854+0000","last_undegraded":"2024-09-13T14:16:25.404854+0000","last_fullsized":"2024-09-13T14:16:25.404854+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398245+0000","last_change":"2024-09-13T14:15:26.555819+0000","last_active":"2024-09-13T14:16:25.398245+0000","last_peered":"2024-09-13T14:16:25.398245+0000","last_clean":"2024-09-13T14:16:25.398245+0000","last_became_active":"2024-09-13T14:15:26.555172+0000","last_became_peered":"2024-09-13T14:15:26.555172+0000","last_unstale":"2024-09-13T14:16:25.398245+0000","last_undegraded":"2024-09-13T14:16:25.398245+0000","last_fullsized":"2024-09-13T14:16:25.398245+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404692+0000","last_change":"2024-09-13T14:15:26.561070+0000","last_active":"2024-09-13T14:16:25.404692+0000","last_peered":"2024-09-13T14:16:25.404692+0000","last_clean":"2024-09-13T14:16:25.404692+0000","last_became_active":"2024-09-13T14:15:26.559771+0000","last_became_peered":"2024-09-13T14:15:26.559771+0000","last_unstale":"2024-09-13T14:16:25.404692+0000","last_undegraded":"2024-09-13T14:16:25.404692+0000","last_fullsized":"2024-09-13T14:16:25.404692+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399852+0000","last_change":"2024-09-13T14:15:26.554841+0000","last_active":"2024-09-13T14:16:25.399852+0000","last_peered":"2024-09-13T14:16:25.399852+0000","last_clean":"2024-09-13T14:16:25.399852+0000","last_became_active":"2024-09-13T14:15:26.553231+0000","last_became_peered":"2024-09-13T14:15:26.553231+0000","last_unstale":"2024-09-13T14:16:25.399852+0000","last_undegraded":"2024-09-13T14:16:25.399852+0000","last_fullsized":"2024-09-13T14:16:25.399852+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427182+0000","last_change":"2024-09-13T14:15:26.551060+0000","last_active":"2024-09-13T14:16:25.427182+0000","last_peered":"2024-09-13T14:16:25.427182+0000","last_clean":"2024-09-13T14:16:25.427182+0000","last_became_active":"2024-09-13T14:15:26.550739+0000","last_became_peered":"2024-09-13T14:15:26.550739+0000","last_unstale":"2024-09-13T14:16:25.427182+0000","last_undegraded":"2024-09-13T14:16:25.427182+0000","last_fullsized":"2024-09-13T14:16:25.427182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402995+0000","last_change":"2024-09-13T14:15:26.559670+0000","last_active":"2024-09-13T14:16:25.402995+0000","last_peered":"2024-09-13T14:16:25.402995+0000","last_clean":"2024-09-13T14:16:25.402995+0000","last_became_active":"2024-09-13T14:15:26.559401+0000","last_became_peered":"2024-09-13T14:15:26.559401+0000","last_unstale":"2024-09-13T14:16:25.402995+0000","last_undegraded":"2024-09-13T14:16:25.402995+0000","last_fullsized":"2024-09-13T14:16:25.402995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399213+0000","last_change":"2024-09-13T14:15:26.570832+0000","last_active":"2024-09-13T14:16:25.399213+0000","last_peered":"2024-09-13T14:16:25.399213+0000","last_clean":"2024-09-13T14:16:25.399213+0000","last_became_active":"2024-09-13T14:15:26.570531+0000","last_became_peered":"2024-09-13T14:15:26.570531+0000","last_unstale":"2024-09-13T14:16:25.399213+0000","last_undegraded":"2024-09-13T14:16:25.399213+0000","last_fullsized":"2024-09-13T14:16:25.399213+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426207+0000","last_change":"2024-09-13T14:15:26.573053+0000","last_active":"2024-09-13T14:16:25.426207+0000","last_peered":"2024-09-13T14:16:25.426207+0000","last_clean":"2024-09-13T14:16:25.426207+0000","last_became_active":"2024-09-13T14:15:26.572783+0000","last_became_peered":"2024-09-13T14:15:26.572783+0000","last_unstale":"2024-09-13T14:16:25.426207+0000","last_undegraded":"2024-09-13T14:16:25.426207+0000","last_fullsized":"2024-09-13T14:16:25.426207+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404968+0000","last_change":"2024-09-13T14:15:26.569346+0000","last_active":"2024-09-13T14:16:25.404968+0000","last_peered":"2024-09-13T14:16:25.404968+0000","last_clean":"2024-09-13T14:16:25.404968+0000","last_became_active":"2024-09-13T14:15:26.568939+0000","last_became_peered":"2024-09-13T14:15:26.568939+0000","last_unstale":"2024-09-13T14:16:25.404968+0000","last_undegraded":"2024-09-13T14:16:25.404968+0000","last_fullsized":"2024-09-13T14:16:25.404968+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397771+0000","last_change":"2024-09-13T14:15:26.562543+0000","last_active":"2024-09-13T14:16:25.397771+0000","last_peered":"2024-09-13T14:16:25.397771+0000","last_clean":"2024-09-13T14:16:25.397771+0000","last_became_active":"2024-09-13T14:15:26.562267+0000","last_became_peered":"2024-09-13T14:15:26.562267+0000","last_unstale":"2024-09-13T14:16:25.397771+0000","last_undegraded":"2024-09-13T14:16:25.397771+0000","last_fullsized":"2024-09-13T14:16:25.397771+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425659+0000","last_change":"2024-09-13T14:15:26.568440+0000","last_active":"2024-09-13T14:16:25.425659+0000","last_peered":"2024-09-13T14:16:25.425659+0000","last_clean":"2024-09-13T14:16:25.425659+0000","last_became_active":"2024-09-13T14:15:26.568169+0000","last_became_peered":"2024-09-13T14:15:26.568169+0000","last_unstale":"2024-09-13T14:16:25.425659+0000","last_undegraded":"2024-09-13T14:16:25.425659+0000","last_fullsized":"2024-09-13T14:16:25.425659+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398398+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:16:25.398398+0000","last_peered":"2024-09-13T14:16:25.398398+0000","last_clean":"2024-09-13T14:16:25.398398+0000","last_became_active":"2024-09-13T14:15:46.963029+0000","last_became_peered":"2024-09-13T14:15:46.963029+0000","last_unstale":"2024-09-13T14:16:25.398398+0000","last_undegraded":"2024-09-13T14:16:25.398398+0000","last_fullsized":"2024-09-13T14:16:25.398398+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400198+0000","last_change":"2024-09-13T14:15:22.541205+0000","last_active":"2024-09-13T14:16:25.400198+0000","last_peered":"2024-09-13T14:16:25.400198+0000","last_clean":"2024-09-13T14:16:25.400198+0000","last_became_active":"2024-09-13T14:15:22.539574+0000","last_became_peered":"2024-09-13T14:15:22.539574+0000","last_unstale":"2024-09-13T14:16:25.400198+0000","last_undegraded":"2024-09-13T14:16:25.400198+0000","last_fullsized":"2024-09-13T14:16:25.400198+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397655+0000","last_change":"2024-09-13T14:15:46.606065+0000","last_active":"2024-09-13T14:16:25.397655+0000","last_peered":"2024-09-13T14:16:25.397655+0000","last_clean":"2024-09-13T14:16:25.397655+0000","last_became_active":"2024-09-13T14:15:46.605583+0000","last_became_peered":"2024-09-13T14:15:46.605583+0000","last_unstale":"2024-09-13T14:16:25.397655+0000","last_undegraded":"2024-09-13T14:16:25.397655+0000","last_fullsized":"2024-09-13T14:16:25.397655+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:21.907855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425880+0000","last_change":"2024-09-13T14:15:22.543444+0000","last_active":"2024-09-13T14:16:25.425880+0000","last_peered":"2024-09-13T14:16:25.425880+0000","last_clean":"2024-09-13T14:16:25.425880+0000","last_became_active":"2024-09-13T14:15:22.543138+0000","last_became_peered":"2024-09-13T14:15:22.543138+0000","last_unstale":"2024-09-13T14:16:25.425880+0000","last_undegraded":"2024-09-13T14:16:25.425880+0000","last_fullsized":"2024-09-13T14:16:25.425880+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399765+0000","last_change":"2024-09-13T14:15:22.541457+0000","last_active":"2024-09-13T14:16:25.399765+0000","last_peered":"2024-09-13T14:16:25.399765+0000","last_clean":"2024-09-13T14:16:25.399765+0000","last_became_active":"2024-09-13T14:15:22.539753+0000","last_became_peered":"2024-09-13T14:15:22.539753+0000","last_unstale":"2024-09-13T14:16:25.399765+0000","last_undegraded":"2024-09-13T14:16:25.399765+0000","last_fullsized":"2024-09-13T14:16:25.399765+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399167+0000","last_change":"2024-09-13T14:15:26.571137+0000","last_active":"2024-09-13T14:16:25.399167+0000","last_peered":"2024-09-13T14:16:25.399167+0000","last_clean":"2024-09-13T14:16:25.399167+0000","last_became_active":"2024-09-13T14:15:26.570884+0000","last_became_peered":"2024-09-13T14:15:26.570884+0000","last_unstale":"2024-09-13T14:16:25.399167+0000","last_undegraded":"2024-09-13T14:16:25.399167+0000","last_fullsized":"2024-09-13T14:16:25.399167+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404803+0000","last_change":"2024-09-13T14:15:22.546875+0000","last_active":"2024-09-13T14:16:25.404803+0000","last_peered":"2024-09-13T14:16:25.404803+0000","last_clean":"2024-09-13T14:16:25.404803+0000","last_became_active":"2024-09-13T14:15:22.545863+0000","last_became_peered":"2024-09-13T14:15:22.545863+0000","last_unstale":"2024-09-13T14:16:25.404803+0000","last_undegraded":"2024-09-13T14:16:25.404803+0000","last_fullsized":"2024-09-13T14:16:25.404803+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402439+0000","last_change":"2024-09-13T14:15:26.560575+0000","last_active":"2024-09-13T14:16:25.402439+0000","last_peered":"2024-09-13T14:16:25.402439+0000","last_clean":"2024-09-13T14:16:25.402439+0000","last_became_active":"2024-09-13T14:15:26.560291+0000","last_became_peered":"2024-09-13T14:15:26.560291+0000","last_unstale":"2024-09-13T14:16:25.402439+0000","last_undegraded":"2024-09-13T14:16:25.402439+0000","last_fullsized":"2024-09-13T14:16:25.402439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427015+0000","last_change":"2024-09-13T14:15:22.546137+0000","last_active":"2024-09-13T14:16:25.427015+0000","last_peered":"2024-09-13T14:16:25.427015+0000","last_clean":"2024-09-13T14:16:25.427015+0000","last_became_active":"2024-09-13T14:15:22.545833+0000","last_became_peered":"2024-09-13T14:15:22.545833+0000","last_unstale":"2024-09-13T14:16:25.427015+0000","last_undegraded":"2024-09-13T14:16:25.427015+0000","last_fullsized":"2024-09-13T14:16:25.427015+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404982+0000","last_change":"2024-09-13T14:15:26.566451+0000","last_active":"2024-09-13T14:16:25.404982+0000","last_peered":"2024-09-13T14:16:25.404982+0000","last_clean":"2024-09-13T14:16:25.404982+0000","last_became_active":"2024-09-13T14:15:26.565363+0000","last_became_peered":"2024-09-13T14:15:26.565363+0000","last_unstale":"2024-09-13T14:16:25.404982+0000","last_undegraded":"2024-09-13T14:16:25.404982+0000","last_fullsized":"2024-09-13T14:16:25.404982+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405218+0000","last_change":"2024-09-13T14:15:22.540966+0000","last_active":"2024-09-13T14:16:25.405218+0000","last_peered":"2024-09-13T14:16:25.405218+0000","last_clean":"2024-09-13T14:16:25.405218+0000","last_became_active":"2024-09-13T14:15:22.540611+0000","last_became_peered":"2024-09-13T14:15:22.540611+0000","last_unstale":"2024-09-13T14:16:25.405218+0000","last_undegraded":"2024-09-13T14:16:25.405218+0000","last_fullsized":"2024-09-13T14:16:25.405218+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426404+0000","last_change":"2024-09-13T14:15:26.573426+0000","last_active":"2024-09-13T14:16:25.426404+0000","last_peered":"2024-09-13T14:16:25.426404+0000","last_clean":"2024-09-13T14:16:25.426404+0000","last_became_active":"2024-09-13T14:15:26.573138+0000","last_became_peered":"2024-09-13T14:15:26.573138+0000","last_unstale":"2024-09-13T14:16:25.426404+0000","last_undegraded":"2024-09-13T14:16:25.426404+0000","last_fullsized":"2024-09-13T14:16:25.426404+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403094+0000","last_change":"2024-09-13T14:15:22.543031+0000","last_active":"2024-09-13T14:16:25.403094+0000","last_peered":"2024-09-13T14:16:25.403094+0000","last_clean":"2024-09-13T14:16:25.403094+0000","last_became_active":"2024-09-13T14:15:22.542739+0000","last_became_peered":"2024-09-13T14:15:22.542739+0000","last_unstale":"2024-09-13T14:16:25.403094+0000","last_undegraded":"2024-09-13T14:16:25.403094+0000","last_fullsized":"2024-09-13T14:16:25.403094+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398640+0000","last_change":"2024-09-13T14:15:26.551575+0000","last_active":"2024-09-13T14:16:25.398640+0000","last_peered":"2024-09-13T14:16:25.398640+0000","last_clean":"2024-09-13T14:16:25.398640+0000","last_became_active":"2024-09-13T14:15:26.550913+0000","last_became_peered":"2024-09-13T14:15:26.550913+0000","last_unstale":"2024-09-13T14:16:25.398640+0000","last_undegraded":"2024-09-13T14:16:25.398640+0000","last_fullsized":"2024-09-13T14:16:25.398640+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426327+0000","last_change":"2024-09-13T14:15:22.539191+0000","last_active":"2024-09-13T14:16:25.426327+0000","last_peered":"2024-09-13T14:16:25.426327+0000","last_clean":"2024-09-13T14:16:25.426327+0000","last_became_active":"2024-09-13T14:15:22.538864+0000","last_became_peered":"2024-09-13T14:15:22.538864+0000","last_unstale":"2024-09-13T14:16:25.426327+0000","last_undegraded":"2024-09-13T14:16:25.426327+0000","last_fullsized":"2024-09-13T14:16:25.426327+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426673+0000","last_change":"2024-09-13T14:15:26.567770+0000","last_active":"2024-09-13T14:16:25.426673+0000","last_peered":"2024-09-13T14:16:25.426673+0000","last_clean":"2024-09-13T14:16:25.426673+0000","last_became_active":"2024-09-13T14:15:26.567513+0000","last_became_peered":"2024-09-13T14:15:26.567513+0000","last_unstale":"2024-09-13T14:16:25.426673+0000","last_undegraded":"2024-09-13T14:16:25.426673+0000","last_fullsized":"2024-09-13T14:16:25.426673+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426237+0000","last_change":"2024-09-13T14:15:22.545520+0000","last_active":"2024-09-13T14:16:25.426237+0000","last_peered":"2024-09-13T14:16:25.426237+0000","last_clean":"2024-09-13T14:16:25.426237+0000","last_became_active":"2024-09-13T14:15:22.545135+0000","last_became_peered":"2024-09-13T14:15:22.545135+0000","last_unstale":"2024-09-13T14:16:25.426237+0000","last_undegraded":"2024-09-13T14:16:25.426237+0000","last_fullsized":"2024-09-13T14:16:25.426237+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426065+0000","last_change":"2024-09-13T14:15:26.575023+0000","last_active":"2024-09-13T14:16:25.426065+0000","last_peered":"2024-09-13T14:16:25.426065+0000","last_clean":"2024-09-13T14:16:25.426065+0000","last_became_active":"2024-09-13T14:15:26.574648+0000","last_became_peered":"2024-09-13T14:15:26.574648+0000","last_unstale":"2024-09-13T14:16:25.426065+0000","last_undegraded":"2024-09-13T14:16:25.426065+0000","last_fullsized":"2024-09-13T14:16:25.426065+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405021+0000","last_change":"2024-09-13T14:15:22.546988+0000","last_active":"2024-09-13T14:16:25.405021+0000","last_peered":"2024-09-13T14:16:25.405021+0000","last_clean":"2024-09-13T14:16:25.405021+0000","last_became_active":"2024-09-13T14:15:22.545255+0000","last_became_peered":"2024-09-13T14:15:22.545255+0000","last_unstale":"2024-09-13T14:16:25.405021+0000","last_undegraded":"2024-09-13T14:16:25.405021+0000","last_fullsized":"2024-09-13T14:16:25.405021+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426955+0000","last_change":"2024-09-13T14:15:26.555682+0000","last_active":"2024-09-13T14:16:25.426955+0000","last_peered":"2024-09-13T14:16:25.426955+0000","last_clean":"2024-09-13T14:16:25.426955+0000","last_became_active":"2024-09-13T14:15:26.555391+0000","last_became_peered":"2024-09-13T14:15:26.555391+0000","last_unstale":"2024-09-13T14:16:25.426955+0000","last_undegraded":"2024-09-13T14:16:25.426955+0000","last_fullsized":"2024-09-13T14:16:25.426955+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425985+0000","last_change":"2024-09-13T14:15:22.542050+0000","last_active":"2024-09-13T14:16:25.425985+0000","last_peered":"2024-09-13T14:16:25.425985+0000","last_clean":"2024-09-13T14:16:25.425985+0000","last_became_active":"2024-09-13T14:15:22.541726+0000","last_became_peered":"2024-09-13T14:15:22.541726+0000","last_unstale":"2024-09-13T14:16:25.425985+0000","last_undegraded":"2024-09-13T14:16:25.425985+0000","last_fullsized":"2024-09-13T14:16:25.425985+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425693+0000","last_change":"2024-09-13T14:15:26.567258+0000","last_active":"2024-09-13T14:16:25.425693+0000","last_peered":"2024-09-13T14:16:25.425693+0000","last_clean":"2024-09-13T14:16:25.425693+0000","last_became_active":"2024-09-13T14:15:26.566924+0000","last_became_peered":"2024-09-13T14:15:26.566924+0000","last_unstale":"2024-09-13T14:16:25.425693+0000","last_undegraded":"2024-09-13T14:16:25.425693+0000","last_fullsized":"2024-09-13T14:16:25.425693+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397636+0000","last_change":"2024-09-13T14:15:22.541058+0000","last_active":"2024-09-13T14:16:25.397636+0000","last_peered":"2024-09-13T14:16:25.397636+0000","last_clean":"2024-09-13T14:16:25.397636+0000","last_became_active":"2024-09-13T14:15:22.540793+0000","last_became_peered":"2024-09-13T14:15:22.540793+0000","last_unstale":"2024-09-13T14:16:25.397636+0000","last_undegraded":"2024-09-13T14:16:25.397636+0000","last_fullsized":"2024-09-13T14:16:25.397636+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426226+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:16:25.426226+0000","last_peered":"2024-09-13T14:16:25.426226+0000","last_clean":"2024-09-13T14:16:25.426226+0000","last_became_active":"2024-09-13T14:15:26.562669+0000","last_became_peered":"2024-09-13T14:15:26.562669+0000","last_unstale":"2024-09-13T14:16:25.426226+0000","last_undegraded":"2024-09-13T14:16:25.426226+0000","last_fullsized":"2024-09-13T14:16:25.426226+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398941+0000","last_change":"2024-09-13T14:15:22.540681+0000","last_active":"2024-09-13T14:16:25.398941+0000","last_peered":"2024-09-13T14:16:25.398941+0000","last_clean":"2024-09-13T14:16:25.398941+0000","last_became_active":"2024-09-13T14:15:22.540386+0000","last_became_peered":"2024-09-13T14:15:22.540386+0000","last_unstale":"2024-09-13T14:16:25.398941+0000","last_undegraded":"2024-09-13T14:16:25.398941+0000","last_fullsized":"2024-09-13T14:16:25.398941+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426867+0000","last_change":"2024-09-13T14:15:26.559281+0000","last_active":"2024-09-13T14:16:25.426867+0000","last_peered":"2024-09-13T14:16:25.426867+0000","last_clean":"2024-09-13T14:16:25.426867+0000","last_became_active":"2024-09-13T14:15:26.558987+0000","last_became_peered":"2024-09-13T14:15:26.558987+0000","last_unstale":"2024-09-13T14:16:25.426867+0000","last_undegraded":"2024-09-13T14:16:25.426867+0000","last_fullsized":"2024-09-13T14:16:25.426867+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398225+0000","last_change":"2024-09-13T14:15:22.544147+0000","last_active":"2024-09-13T14:16:25.398225+0000","last_peered":"2024-09-13T14:16:25.398225+0000","last_clean":"2024-09-13T14:16:25.398225+0000","last_became_active":"2024-09-13T14:15:22.543782+0000","last_became_peered":"2024-09-13T14:15:22.543782+0000","last_unstale":"2024-09-13T14:16:25.398225+0000","last_undegraded":"2024-09-13T14:16:25.398225+0000","last_fullsized":"2024-09-13T14:16:25.398225+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427487+0000","last_change":"2024-09-13T14:15:26.570603+0000","last_active":"2024-09-13T14:16:25.427487+0000","last_peered":"2024-09-13T14:16:25.427487+0000","last_clean":"2024-09-13T14:16:25.427487+0000","last_became_active":"2024-09-13T14:15:26.569357+0000","last_became_peered":"2024-09-13T14:15:26.569357+0000","last_unstale":"2024-09-13T14:16:25.427487+0000","last_undegraded":"2024-09-13T14:16:25.427487+0000","last_fullsized":"2024-09-13T14:16:25.427487+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403205+0000","last_change":"2024-09-13T14:15:22.546363+0000","last_active":"2024-09-13T14:16:25.403205+0000","last_peered":"2024-09-13T14:16:25.403205+0000","last_clean":"2024-09-13T14:16:25.403205+0000","last_became_active":"2024-09-13T14:15:22.544417+0000","last_became_peered":"2024-09-13T14:15:22.544417+0000","last_unstale":"2024-09-13T14:16:25.403205+0000","last_undegraded":"2024-09-13T14:16:25.403205+0000","last_fullsized":"2024-09-13T14:16:25.403205+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404958+0000","last_change":"2024-09-13T14:15:26.572445+0000","last_active":"2024-09-13T14:16:25.404958+0000","last_peered":"2024-09-13T14:16:25.404958+0000","last_clean":"2024-09-13T14:16:25.404958+0000","last_became_active":"2024-09-13T14:15:26.572166+0000","last_became_peered":"2024-09-13T14:15:26.572166+0000","last_unstale":"2024-09-13T14:16:25.404958+0000","last_undegraded":"2024-09-13T14:16:25.404958+0000","last_fullsized":"2024-09-13T14:16:25.404958+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398707+0000","last_change":"2024-09-13T14:15:22.542920+0000","last_active":"2024-09-13T14:16:25.398707+0000","last_peered":"2024-09-13T14:16:25.398707+0000","last_clean":"2024-09-13T14:16:25.398707+0000","last_became_active":"2024-09-13T14:15:22.542641+0000","last_became_peered":"2024-09-13T14:15:22.542641+0000","last_unstale":"2024-09-13T14:16:25.398707+0000","last_undegraded":"2024-09-13T14:16:25.398707+0000","last_fullsized":"2024-09-13T14:16:25.398707+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426854+0000","last_change":"2024-09-13T14:15:26.572195+0000","last_active":"2024-09-13T14:16:25.426854+0000","last_peered":"2024-09-13T14:16:25.426854+0000","last_clean":"2024-09-13T14:16:25.426854+0000","last_became_active":"2024-09-13T14:15:26.571003+0000","last_became_peered":"2024-09-13T14:15:26.571003+0000","last_unstale":"2024-09-13T14:16:25.426854+0000","last_undegraded":"2024-09-13T14:16:25.426854+0000","last_fullsized":"2024-09-13T14:16:25.426854+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:33.691439+0000","last_change":"2024-09-13T14:16:25.426886+0000","last_active":"2024-09-13T14:16:33.691439+0000","last_peered":"2024-09-13T14:16:33.691439+0000","last_clean":"2024-09-13T14:16:33.691439+0000","last_became_active":"2024-09-13T14:16:21.376845+0000","last_became_peered":"2024-09-13T14:16:21.376845+0000","last_unstale":"2024-09-13T14:16:33.691439+0000","last_undegraded":"2024-09-13T14:16:33.691439+0000","last_fullsized":"2024-09-13T14:16:33.691439+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:10.386795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055469799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"60'79","reported_seq":57,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427505+0000","last_change":"2024-09-13T14:14:57.479980+0000","last_active":"2024-09-13T14:16:25.427505+0000","last_peered":"2024-09-13T14:16:25.427505+0000","last_clean":"2024-09-13T14:16:25.427505+0000","last_became_active":"2024-09-13T14:14:57.475298+0000","last_became_peered":"2024-09-13T14:14:57.475298+0000","last_unstale":"2024-09-13T14:16:25.427505+0000","last_undegraded":"2024-09-13T14:16:25.427505+0000","last_fullsized":"2024-09-13T14:16:25.427505+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_clean_scrub_stamp":"2024-09-13T14:12:56.059430+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404852+0000","last_change":"2024-09-13T14:15:22.542042+0000","last_active":"2024-09-13T14:16:25.404852+0000","last_peered":"2024-09-13T14:16:25.404852+0000","last_clean":"2024-09-13T14:16:25.404852+0000","last_became_active":"2024-09-13T14:15:22.541756+0000","last_became_peered":"2024-09-13T14:15:22.541756+0000","last_unstale":"2024-09-13T14:16:25.404852+0000","last_undegraded":"2024-09-13T14:16:25.404852+0000","last_fullsized":"2024-09-13T14:16:25.404852+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398634+0000","last_change":"2024-09-13T14:15:26.565510+0000","last_active":"2024-09-13T14:16:25.398634+0000","last_peered":"2024-09-13T14:16:25.398634+0000","last_clean":"2024-09-13T14:16:25.398634+0000","last_became_active":"2024-09-13T14:15:26.565213+0000","last_became_peered":"2024-09-13T14:15:26.565213+0000","last_unstale":"2024-09-13T14:16:25.398634+0000","last_undegraded":"2024-09-13T14:16:25.398634+0000","last_fullsized":"2024-09-13T14:16:25.398634+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.002691+0000","last_change":"2024-09-13T14:16:25.426290+0000","last_active":"2024-09-13T14:16:34.002691+0000","last_peered":"2024-09-13T14:16:34.002691+0000","last_clean":"2024-09-13T14:16:34.002691+0000","last_became_active":"2024-09-13T14:16:22.202033+0000","last_became_peered":"2024-09-13T14:16:22.202033+0000","last_unstale":"2024-09-13T14:16:34.002691+0000","last_undegraded":"2024-09-13T14:16:34.002691+0000","last_fullsized":"2024-09-13T14:16:34.002691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:48.139777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071034400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427357+0000","last_change":"2024-09-13T14:15:22.550379+0000","last_active":"2024-09-13T14:16:25.427357+0000","last_peered":"2024-09-13T14:16:25.427357+0000","last_clean":"2024-09-13T14:16:25.427357+0000","last_became_active":"2024-09-13T14:15:22.549995+0000","last_became_peered":"2024-09-13T14:15:22.549995+0000","last_unstale":"2024-09-13T14:16:25.427357+0000","last_undegraded":"2024-09-13T14:16:25.427357+0000","last_fullsized":"2024-09-13T14:16:25.427357+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425608+0000","last_change":"2024-09-13T14:15:46.965011+0000","last_active":"2024-09-13T14:16:25.425608+0000","last_peered":"2024-09-13T14:16:25.425608+0000","last_clean":"2024-09-13T14:16:25.425608+0000","last_became_active":"2024-09-13T14:15:46.964646+0000","last_became_peered":"2024-09-13T14:15:46.964646+0000","last_unstale":"2024-09-13T14:16:25.425608+0000","last_undegraded":"2024-09-13T14:16:25.425608+0000","last_fullsized":"2024-09-13T14:16:25.425608+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:07.136822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.174222+0000","last_change":"2024-09-13T14:16:25.403479+0000","last_active":"2024-09-13T14:16:34.174222+0000","last_peered":"2024-09-13T14:16:34.174222+0000","last_clean":"2024-09-13T14:16:34.174222+0000","last_became_active":"2024-09-13T14:16:21.605160+0000","last_became_peered":"2024-09-13T14:16:21.605160+0000","last_unstale":"2024-09-13T14:16:34.174222+0000","last_undegraded":"2024-09-13T14:16:34.174222+0000","last_fullsized":"2024-09-13T14:16:34.174222+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:48.959952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034207600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398529+0000","last_change":"2024-09-13T14:15:22.544154+0000","last_active":"2024-09-13T14:16:25.398529+0000","last_peered":"2024-09-13T14:16:25.398529+0000","last_clean":"2024-09-13T14:16:25.398529+0000","last_became_active":"2024-09-13T14:15:22.543880+0000","last_became_peered":"2024-09-13T14:15:22.543880+0000","last_unstale":"2024-09-13T14:16:25.398529+0000","last_undegraded":"2024-09-13T14:16:25.398529+0000","last_fullsized":"2024-09-13T14:16:25.398529+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402399+0000","last_change":"2024-09-13T14:15:26.553134+0000","last_active":"2024-09-13T14:16:25.402399+0000","last_peered":"2024-09-13T14:16:25.402399+0000","last_clean":"2024-09-13T14:16:25.402399+0000","last_became_active":"2024-09-13T14:15:26.552054+0000","last_became_peered":"2024-09-13T14:15:26.552054+0000","last_unstale":"2024-09-13T14:16:25.402399+0000","last_undegraded":"2024-09-13T14:16:25.402399+0000","last_fullsized":"2024-09-13T14:16:25.402399+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.173862+0000","last_change":"2024-09-13T14:16:25.403217+0000","last_active":"2024-09-13T14:16:34.173862+0000","last_peered":"2024-09-13T14:16:34.173862+0000","last_clean":"2024-09-13T14:16:34.173862+0000","last_became_active":"2024-09-13T14:16:22.202029+0000","last_became_peered":"2024-09-13T14:16:22.202029+0000","last_unstale":"2024-09-13T14:16:34.173862+0000","last_undegraded":"2024-09-13T14:16:34.173862+0000","last_fullsized":"2024-09-13T14:16:34.173862+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:48.284490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000388303,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405168+0000","last_change":"2024-09-13T14:15:22.545901+0000","last_active":"2024-09-13T14:16:25.405168+0000","last_peered":"2024-09-13T14:16:25.405168+0000","last_clean":"2024-09-13T14:16:25.405168+0000","last_became_active":"2024-09-13T14:15:22.545610+0000","last_became_peered":"2024-09-13T14:15:22.545610+0000","last_unstale":"2024-09-13T14:16:25.405168+0000","last_undegraded":"2024-09-13T14:16:25.405168+0000","last_fullsized":"2024-09-13T14:16:25.405168+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398541+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:16:25.398541+0000","last_peered":"2024-09-13T14:16:25.398541+0000","last_clean":"2024-09-13T14:16:25.398541+0000","last_became_active":"2024-09-13T14:15:26.562679+0000","last_became_peered":"2024-09-13T14:15:26.562679+0000","last_unstale":"2024-09-13T14:16:25.398541+0000","last_undegraded":"2024-09-13T14:16:25.398541+0000","last_fullsized":"2024-09-13T14:16:25.398541+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.173690+0000","last_change":"2024-09-13T14:16:25.403491+0000","last_active":"2024-09-13T14:16:34.173690+0000","last_peered":"2024-09-13T14:16:34.173690+0000","last_clean":"2024-09-13T14:16:34.173690+0000","last_became_active":"2024-09-13T14:16:21.605236+0000","last_became_peered":"2024-09-13T14:16:21.605236+0000","last_unstale":"2024-09-13T14:16:34.173690+0000","last_undegraded":"2024-09-13T14:16:34.173690+0000","last_fullsized":"2024-09-13T14:16:34.173690+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:47.051775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097564099999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403019+0000","last_change":"2024-09-13T14:15:22.549591+0000","last_active":"2024-09-13T14:16:25.403019+0000","last_peered":"2024-09-13T14:16:25.403019+0000","last_clean":"2024-09-13T14:16:25.403019+0000","last_became_active":"2024-09-13T14:15:22.549305+0000","last_became_peered":"2024-09-13T14:15:22.549305+0000","last_unstale":"2024-09-13T14:16:25.403019+0000","last_undegraded":"2024-09-13T14:16:25.403019+0000","last_fullsized":"2024-09-13T14:16:25.403019+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405183+0000","last_change":"2024-09-13T14:15:26.562976+0000","last_active":"2024-09-13T14:16:25.405183+0000","last_peered":"2024-09-13T14:16:25.405183+0000","last_clean":"2024-09-13T14:16:25.405183+0000","last_became_active":"2024-09-13T14:15:26.561391+0000","last_became_peered":"2024-09-13T14:15:26.561391+0000","last_unstale":"2024-09-13T14:16:25.405183+0000","last_undegraded":"2024-09-13T14:16:25.405183+0000","last_fullsized":"2024-09-13T14:16:25.405183+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.002319+0000","last_change":"2024-09-13T14:16:25.426585+0000","last_active":"2024-09-13T14:16:34.002319+0000","last_peered":"2024-09-13T14:16:34.002319+0000","last_clean":"2024-09-13T14:16:34.002319+0000","last_became_active":"2024-09-13T14:16:21.604772+0000","last_became_peered":"2024-09-13T14:16:21.604772+0000","last_unstale":"2024-09-13T14:16:34.002319+0000","last_undegraded":"2024-09-13T14:16:34.002319+0000","last_fullsized":"2024-09-13T14:16:34.002319+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:51.564452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000446272,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427204+0000","last_change":"2024-09-13T14:15:22.550059+0000","last_active":"2024-09-13T14:16:25.427204+0000","last_peered":"2024-09-13T14:16:25.427204+0000","last_clean":"2024-09-13T14:16:25.427204+0000","last_became_active":"2024-09-13T14:15:22.549682+0000","last_became_peered":"2024-09-13T14:15:22.549682+0000","last_unstale":"2024-09-13T14:16:25.427204+0000","last_undegraded":"2024-09-13T14:16:25.427204+0000","last_fullsized":"2024-09-13T14:16:25.427204+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402936+0000","last_change":"2024-09-13T14:15:26.565253+0000","last_active":"2024-09-13T14:16:25.402936+0000","last_peered":"2024-09-13T14:16:25.402936+0000","last_clean":"2024-09-13T14:16:25.402936+0000","last_became_active":"2024-09-13T14:15:26.564972+0000","last_became_peered":"2024-09-13T14:15:26.564972+0000","last_unstale":"2024-09-13T14:16:25.402936+0000","last_undegraded":"2024-09-13T14:16:25.402936+0000","last_fullsized":"2024-09-13T14:16:25.402936+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:33.691085+0000","last_change":"2024-09-13T14:16:25.399510+0000","last_active":"2024-09-13T14:16:33.691085+0000","last_peered":"2024-09-13T14:16:33.691085+0000","last_clean":"2024-09-13T14:16:33.691085+0000","last_became_active":"2024-09-13T14:16:21.374161+0000","last_became_peered":"2024-09-13T14:16:21.374161+0000","last_unstale":"2024-09-13T14:16:33.691085+0000","last_undegraded":"2024-09-13T14:16:33.691085+0000","last_fullsized":"2024-09-13T14:16:33.691085+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:24:25.293852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049448099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403350+0000","last_change":"2024-09-13T14:15:22.547223+0000","last_active":"2024-09-13T14:16:25.403350+0000","last_peered":"2024-09-13T14:16:25.403350+0000","last_clean":"2024-09-13T14:16:25.403350+0000","last_became_active":"2024-09-13T14:15:22.546934+0000","last_became_peered":"2024-09-13T14:15:22.546934+0000","last_unstale":"2024-09-13T14:16:25.403350+0000","last_undegraded":"2024-09-13T14:16:25.403350+0000","last_fullsized":"2024-09-13T14:16:25.403350+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399968+0000","last_change":"2024-09-13T14:15:26.571004+0000","last_active":"2024-09-13T14:16:25.399968+0000","last_peered":"2024-09-13T14:16:25.399968+0000","last_clean":"2024-09-13T14:16:25.399968+0000","last_became_active":"2024-09-13T14:15:26.570753+0000","last_became_peered":"2024-09-13T14:15:26.570753+0000","last_unstale":"2024-09-13T14:16:25.399968+0000","last_undegraded":"2024-09-13T14:16:25.399968+0000","last_fullsized":"2024-09-13T14:16:25.399968+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.009686+0000","last_change":"2024-09-13T14:16:25.399066+0000","last_active":"2024-09-13T14:16:34.009686+0000","last_peered":"2024-09-13T14:16:34.009686+0000","last_clean":"2024-09-13T14:16:34.009686+0000","last_became_active":"2024-09-13T14:16:21.605337+0000","last_became_peered":"2024-09-13T14:16:21.605337+0000","last_unstale":"2024-09-13T14:16:34.009686+0000","last_undegraded":"2024-09-13T14:16:34.009686+0000","last_fullsized":"2024-09-13T14:16:34.009686+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:20.334474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058535100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427115+0000","last_change":"2024-09-13T14:15:22.545342+0000","last_active":"2024-09-13T14:16:25.427115+0000","last_peered":"2024-09-13T14:16:25.427115+0000","last_clean":"2024-09-13T14:16:25.427115+0000","last_became_active":"2024-09-13T14:15:22.545067+0000","last_became_peered":"2024-09-13T14:15:22.545067+0000","last_unstale":"2024-09-13T14:16:25.427115+0000","last_undegraded":"2024-09-13T14:16:25.427115+0000","last_fullsized":"2024-09-13T14:16:25.427115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397966+0000","last_change":"2024-09-13T14:15:46.963528+0000","last_active":"2024-09-13T14:16:25.397966+0000","last_peered":"2024-09-13T14:16:25.397966+0000","last_clean":"2024-09-13T14:16:25.397966+0000","last_became_active":"2024-09-13T14:15:46.963216+0000","last_became_peered":"2024-09-13T14:15:46.963216+0000","last_unstale":"2024-09-13T14:16:25.397966+0000","last_undegraded":"2024-09-13T14:16:25.397966+0000","last_fullsized":"2024-09-13T14:16:25.397966+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:29.430917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427235+0000","last_change":"2024-09-13T14:15:22.550293+0000","last_active":"2024-09-13T14:16:25.427235+0000","last_peered":"2024-09-13T14:16:25.427235+0000","last_clean":"2024-09-13T14:16:25.427235+0000","last_became_active":"2024-09-13T14:15:22.549735+0000","last_became_peered":"2024-09-13T14:15:22.549735+0000","last_unstale":"2024-09-13T14:16:25.427235+0000","last_undegraded":"2024-09-13T14:16:25.427235+0000","last_fullsized":"2024-09-13T14:16:25.427235+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398463+0000","last_change":"2024-09-13T14:15:26.556457+0000","last_active":"2024-09-13T14:16:25.398463+0000","last_peered":"2024-09-13T14:16:25.398463+0000","last_clean":"2024-09-13T14:16:25.398463+0000","last_became_active":"2024-09-13T14:15:26.555684+0000","last_became_peered":"2024-09-13T14:15:26.555684+0000","last_unstale":"2024-09-13T14:16:25.398463+0000","last_undegraded":"2024-09-13T14:16:25.398463+0000","last_fullsized":"2024-09-13T14:16:25.398463+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402595+0000","last_change":"2024-09-13T14:15:22.549793+0000","last_active":"2024-09-13T14:16:25.402595+0000","last_peered":"2024-09-13T14:16:25.402595+0000","last_clean":"2024-09-13T14:16:25.402595+0000","last_became_active":"2024-09-13T14:15:22.549291+0000","last_became_peered":"2024-09-13T14:15:22.549291+0000","last_unstale":"2024-09-13T14:16:25.402595+0000","last_undegraded":"2024-09-13T14:16:25.402595+0000","last_fullsized":"2024-09-13T14:16:25.402595+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399236+0000","last_change":"2024-09-13T14:15:26.554955+0000","last_active":"2024-09-13T14:16:25.399236+0000","last_peered":"2024-09-13T14:16:25.399236+0000","last_clean":"2024-09-13T14:16:25.399236+0000","last_became_active":"2024-09-13T14:15:26.553125+0000","last_became_peered":"2024-09-13T14:15:26.553125+0000","last_unstale":"2024-09-13T14:16:25.399236+0000","last_undegraded":"2024-09-13T14:16:25.399236+0000","last_fullsized":"2024-09-13T14:16:25.399236+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402834+0000","last_change":"2024-09-13T14:15:22.549798+0000","last_active":"2024-09-13T14:16:25.402834+0000","last_peered":"2024-09-13T14:16:25.402834+0000","last_clean":"2024-09-13T14:16:25.402834+0000","last_became_active":"2024-09-13T14:15:22.549508+0000","last_became_peered":"2024-09-13T14:15:22.549508+0000","last_unstale":"2024-09-13T14:16:25.402834+0000","last_undegraded":"2024-09-13T14:16:25.402834+0000","last_fullsized":"2024-09-13T14:16:25.402834+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400073+0000","last_change":"2024-09-13T14:15:26.571783+0000","last_active":"2024-09-13T14:16:25.400073+0000","last_peered":"2024-09-13T14:16:25.400073+0000","last_clean":"2024-09-13T14:16:25.400073+0000","last_became_active":"2024-09-13T14:15:26.571523+0000","last_became_peered":"2024-09-13T14:15:26.571523+0000","last_unstale":"2024-09-13T14:16:25.400073+0000","last_undegraded":"2024-09-13T14:16:25.400073+0000","last_fullsized":"2024-09-13T14:16:25.400073+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427718+0000","last_change":"2024-09-13T14:15:22.549324+0000","last_active":"2024-09-13T14:16:25.427718+0000","last_peered":"2024-09-13T14:16:25.427718+0000","last_clean":"2024-09-13T14:16:25.427718+0000","last_became_active":"2024-09-13T14:15:22.549022+0000","last_became_peered":"2024-09-13T14:15:22.549022+0000","last_unstale":"2024-09-13T14:16:25.427718+0000","last_undegraded":"2024-09-13T14:16:25.427718+0000","last_fullsized":"2024-09-13T14:16:25.427718+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425792+0000","last_change":"2024-09-13T14:15:26.549648+0000","last_active":"2024-09-13T14:16:25.425792+0000","last_peered":"2024-09-13T14:16:25.425792+0000","last_clean":"2024-09-13T14:16:25.425792+0000","last_became_active":"2024-09-13T14:15:26.549328+0000","last_became_peered":"2024-09-13T14:15:26.549328+0000","last_unstale":"2024-09-13T14:16:25.425792+0000","last_undegraded":"2024-09-13T14:16:25.425792+0000","last_fullsized":"2024-09-13T14:16:25.425792+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405098+0000","last_change":"2024-09-13T14:15:26.573588+0000","last_active":"2024-09-13T14:16:25.405098+0000","last_peered":"2024-09-13T14:16:25.405098+0000","last_clean":"2024-09-13T14:16:25.405098+0000","last_became_active":"2024-09-13T14:15:26.572830+0000","last_became_peered":"2024-09-13T14:15:26.572830+0000","last_unstale":"2024-09-13T14:16:25.405098+0000","last_undegraded":"2024-09-13T14:16:25.405098+0000","last_fullsized":"2024-09-13T14:16:25.405098+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426524+0000","last_change":"2024-09-13T14:15:22.550180+0000","last_active":"2024-09-13T14:16:25.426524+0000","last_peered":"2024-09-13T14:16:25.426524+0000","last_clean":"2024-09-13T14:16:25.426524+0000","last_became_active":"2024-09-13T14:15:22.549754+0000","last_became_peered":"2024-09-13T14:15:22.549754+0000","last_unstale":"2024-09-13T14:16:25.426524+0000","last_undegraded":"2024-09-13T14:16:25.426524+0000","last_fullsized":"2024-09-13T14:16:25.426524+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398480+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:16:25.398480+0000","last_peered":"2024-09-13T14:16:25.398480+0000","last_clean":"2024-09-13T14:16:25.398480+0000","last_became_active":"2024-09-13T14:15:46.963081+0000","last_became_peered":"2024-09-13T14:15:46.963081+0000","last_unstale":"2024-09-13T14:16:25.398480+0000","last_undegraded":"2024-09-13T14:16:25.398480+0000","last_fullsized":"2024-09-13T14:16:25.398480+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:51.607549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426984+0000","last_change":"2024-09-13T14:15:22.550265+0000","last_active":"2024-09-13T14:16:25.426984+0000","last_peered":"2024-09-13T14:16:25.426984+0000","last_clean":"2024-09-13T14:16:25.426984+0000","last_became_active":"2024-09-13T14:15:22.549179+0000","last_became_peered":"2024-09-13T14:15:22.549179+0000","last_unstale":"2024-09-13T14:16:25.426984+0000","last_undegraded":"2024-09-13T14:16:25.426984+0000","last_fullsized":"2024-09-13T14:16:25.426984+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426376+0000","last_change":"2024-09-13T14:15:26.575732+0000","last_active":"2024-09-13T14:16:25.426376+0000","last_peered":"2024-09-13T14:16:25.426376+0000","last_clean":"2024-09-13T14:16:25.426376+0000","last_became_active":"2024-09-13T14:15:26.575235+0000","last_became_peered":"2024-09-13T14:15:26.575235+0000","last_unstale":"2024-09-13T14:16:25.426376+0000","last_undegraded":"2024-09-13T14:16:25.426376+0000","last_fullsized":"2024-09-13T14:16:25.426376+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400237+0000","last_change":"2024-09-13T14:15:22.542901+0000","last_active":"2024-09-13T14:16:25.400237+0000","last_peered":"2024-09-13T14:16:25.400237+0000","last_clean":"2024-09-13T14:16:25.400237+0000","last_became_active":"2024-09-13T14:15:22.542604+0000","last_became_peered":"2024-09-13T14:15:22.542604+0000","last_unstale":"2024-09-13T14:16:25.400237+0000","last_undegraded":"2024-09-13T14:16:25.400237+0000","last_fullsized":"2024-09-13T14:16:25.400237+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400092+0000","last_change":"2024-09-13T14:15:26.555625+0000","last_active":"2024-09-13T14:16:25.400092+0000","last_peered":"2024-09-13T14:16:25.400092+0000","last_clean":"2024-09-13T14:16:25.400092+0000","last_became_active":"2024-09-13T14:15:26.554633+0000","last_became_peered":"2024-09-13T14:15:26.554633+0000","last_unstale":"2024-09-13T14:16:25.400092+0000","last_undegraded":"2024-09-13T14:16:25.400092+0000","last_fullsized":"2024-09-13T14:16:25.400092+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":44,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426535+0000","last_change":"2024-09-13T14:15:46.609525+0000","last_active":"2024-09-13T14:16:25.426535+0000","last_peered":"2024-09-13T14:16:25.426535+0000","last_clean":"2024-09-13T14:16:25.426535+0000","last_became_active":"2024-09-13T14:15:46.605752+0000","last_became_peered":"2024-09-13T14:15:46.605752+0000","last_unstale":"2024-09-13T14:16:25.426535+0000","last_undegraded":"2024-09-13T14:16:25.426535+0000","last_fullsized":"2024-09-13T14:16:25.426535+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:29.004573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430240,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":900361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.67400000000000004,"5min":0.749,"15min":0.749},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.56000000000000005,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63800000000000001,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.57699999999999996}]},{"osd":2,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.627},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.56100000000000005,"15min":0.56899999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.69299999999999995,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.60899999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58099999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64800000000000002,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51700000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57799999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46600000000000003,"15min":0.47399999999999998},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66900000000000004,"5min":0.745,"15min":0.745},"last":0.67400000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.48899999999999999,"15min":0.497},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65500000000000003,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.44,"5min":0.49299999999999999,"15min":0.502},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64000000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.73399999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.503,"15min":0.50900000000000001},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68100000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.57299999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.69999999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626468,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":902387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54400000000000004,"15min":0.55200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.59099999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75900000000000001,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.71399999999999997}]},{"osd":2,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53500000000000003,"15min":0.54000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78000000000000003,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.66800000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.63700000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28199999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60299999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.52800000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.66100000000000003,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47499999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:17:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.495,"15min":0.495},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.69599999999999995,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.496},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.62}]}]},{"osd":5,"up_from":36,"seq":154618822696,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":311792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51900000000000002,"15min":0.53200000000000003},"min":{"1min":0.40600000000000003,"5min":0.312,"15min":0.312},"max":{"1min":0.64600000000000002,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54300000000000004,"15min":0.54200000000000004},"min":{"1min":0.35999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70899999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.41699999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.35299999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69899999999999995,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.372},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.36799999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.50600000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.39700000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67900000000000005,"5min":0.755,"15min":0.755},"last":0.435},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73799999999999999,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60999999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52700000000000002,"15min":0.53900000000000003},"min":{"1min":0.36499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65400000000000003,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.68400000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.56100000000000005}]},{"osd":4,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.40799999999999997,"15min":0.39400000000000002},"min":{"1min":0.307,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66400000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.35999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.53800000000000003,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.45100000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.47399999999999998,"15min":0.47899999999999998},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.46400000000000002,"15min":0.46600000000000003},"min":{"1min":0.28899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52400000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.51700000000000002,"15min":0.52900000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61299999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71299999999999997,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018924,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56000000000000005,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.63700000000000001,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68700000000000006,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.45000000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65700000000000003,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.56200000000000006,"15min":0.57599999999999996},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66800000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.628,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.626},{"interface":"front","average":{"1min":0.504,"5min":0.56200000000000006,"15min":0.57099999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.61599999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.58499999999999996,"15min":0.59499999999999997},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.66100000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.501},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63400000000000001,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:17:15 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54100000000000004,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.38},{"interface":"front","average":{"1min":0.434,"5min":0.39000000000000001,"15min":0.375},"min":{"1min":0.26300000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.40899999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.41299999999999998,"15min":0.41499999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.433}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65000000000000002,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.39200000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":312417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":0.623,"5min":0.628,"15min":0.628},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.222,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.44400000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.58399999999999996,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.39200000000000002,"15min":0.38800000000000001},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.32700000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.371},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.40500000000000003,"15min":0.40400000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.40500000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:16:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59499999999999997,"15min":0.57699999999999996},"min":{"1min":0.442,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86499999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.59999999999999998,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.86599999999999999,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.63600000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:17:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.54900000000000004},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.874,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57999999999999996,"15min":0.56999999999999995},"min":{"1min":0.435,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.74299999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:16:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.46400000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95299999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.56200000000000006}]},{"osd":7,"last update":"Fri Sep 13 14:16:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.433,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.88,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.63700000000000001,"15min":0.64300000000000002},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.85899999999999999,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58199999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":692224,"data_stored":310541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.67900000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.39100000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.57199999999999995,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.40500000000000003},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.41499999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:16:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45700000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73099999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.377},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64800000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.46000000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:16:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78800000000000003,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.57999999999999996,"15min":0.58099999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71499999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.751}]},{"osd":5,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.63300000000000001,"15min":0.64200000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80700000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.73299999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:16:33 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60799999999999998,"15min":0.60699999999999998},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63900000000000001,"15min":0.64400000000000002},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70499999999999996}]},{"osd":7,"last update":"Fri Sep 13 14:16:57 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.65900000000000003,"15min":0.66500000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84999999999999998,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83999999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68700000000000006}]}]},{"osd":1,"up_from":13,"seq":55834574908,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":692224,"data_stored":309520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:37 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.38700000000000001,"15min":0.34699999999999998},"min":{"1min":0.309,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.54900000000000004,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.438},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.434,"15min":0.379},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.67600000000000005,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.60099999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:17:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.39900000000000002,"15min":0.375},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.64400000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41799999999999998,"15min":0.39400000000000002},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.38500000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:16:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":0.63700000000000001,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61099999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.35799999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:16:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58699999999999997,"15min":0.57399999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.56499999999999995,"15min":0.55400000000000005},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.53000000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:17:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.63700000000000001,"15min":0.65200000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Sep 13 14:16:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.63900000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.65700000000000003,"15min":0.66000000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.85399999999999998,"5min":0.87,"15min":0.87},"last":0.57899999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738433,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":901055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.27800000000000002,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":0.54000000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.37,"15min":0.34899999999999998},"min":{"1min":0.24299999999999999,"5min":0.184,"15min":0.184},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.43099999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:17:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39600000000000002,"15min":0.38400000000000001},"min":{"1min":0.29899999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.46800000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48699999999999999,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.45500000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:16:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.47299999999999998,"15min":0.47199999999999998},"min":{"1min":0.34699999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65900000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.497},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.57999999999999996,"5min":0.625,"15min":0.625},"last":0.41699999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:16:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.55600000000000005,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79100000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.80600000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:16:15 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55200000000000005,"15min":0.54500000000000004},"min":{"1min":0.44500000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.73899999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.78200000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.47499999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.76500000000000001,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.84699999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:17:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58999999999999997,"15min":0.59299999999999997},"min":{"1min":0.438,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.628},{"interface":"front","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.47899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.82599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:17:23.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:17:23.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:23 smithi026 ceph-mon[39182]: from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:23.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:23 smithi026 ceph-mon[27770]: from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:23 smithi145 ceph-mon[31057]: from='client.14709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:23.757 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:24 smithi145 ceph-mon[31057]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:24.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:24 smithi026 ceph-mon[39182]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:24.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:24 smithi026 ceph-mon[27770]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:25.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:25.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:25 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:25 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:25 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:25 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:25 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:25 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:26.439 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:26.440 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[39182]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[39182]: Reconfiguring daemon alertmanager.a on smithi026 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[39182]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[27770]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[27770]: Reconfiguring daemon alertmanager.a on smithi026 2024-09-13T14:17:26.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:26 smithi026 ceph-mon[27770]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:26 smithi145 ceph-mon[31057]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:17:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:26 smithi145 ceph-mon[31057]: Reconfiguring daemon alertmanager.a on smithi026 2024-09-13T14:17:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:26 smithi145 ceph-mon[31057]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:27.728 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":44,"stamp":"2024-09-13T14:17:25.376466+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":590938,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386048,"kb_used_data":7204,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575168,"statfs":{"total":767960285184,"available":765516972032,"internally_reserved":0,"allocated":7376896,"data_stored":4258759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002975"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427311+0000","last_change":"2024-09-13T14:15:26.568393+0000","last_active":"2024-09-13T14:16:25.427311+0000","last_peered":"2024-09-13T14:16:25.427311+0000","last_clean":"2024-09-13T14:16:25.427311+0000","last_became_active":"2024-09-13T14:15:26.568075+0000","last_became_peered":"2024-09-13T14:15:26.568075+0000","last_unstale":"2024-09-13T14:16:25.427311+0000","last_undegraded":"2024-09-13T14:16:25.427311+0000","last_fullsized":"2024-09-13T14:16:25.427311+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403288+0000","last_change":"2024-09-13T14:15:26.563624+0000","last_active":"2024-09-13T14:16:25.403288+0000","last_peered":"2024-09-13T14:16:25.403288+0000","last_clean":"2024-09-13T14:16:25.403288+0000","last_became_active":"2024-09-13T14:15:26.563364+0000","last_became_peered":"2024-09-13T14:15:26.563364+0000","last_unstale":"2024-09-13T14:16:25.403288+0000","last_undegraded":"2024-09-13T14:16:25.403288+0000","last_fullsized":"2024-09-13T14:16:25.403288+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426637+0000","last_change":"2024-09-13T14:15:26.566586+0000","last_active":"2024-09-13T14:16:25.426637+0000","last_peered":"2024-09-13T14:16:25.426637+0000","last_clean":"2024-09-13T14:16:25.426637+0000","last_became_active":"2024-09-13T14:15:26.566331+0000","last_became_peered":"2024-09-13T14:15:26.566331+0000","last_unstale":"2024-09-13T14:16:25.426637+0000","last_undegraded":"2024-09-13T14:16:25.426637+0000","last_fullsized":"2024-09-13T14:16:25.426637+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404976+0000","last_change":"2024-09-13T14:15:26.564769+0000","last_active":"2024-09-13T14:16:25.404976+0000","last_peered":"2024-09-13T14:16:25.404976+0000","last_clean":"2024-09-13T14:16:25.404976+0000","last_became_active":"2024-09-13T14:15:26.564496+0000","last_became_peered":"2024-09-13T14:15:26.564496+0000","last_unstale":"2024-09-13T14:16:25.404976+0000","last_undegraded":"2024-09-13T14:16:25.404976+0000","last_fullsized":"2024-09-13T14:16:25.404976+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427347+0000","last_change":"2024-09-13T14:15:26.555592+0000","last_active":"2024-09-13T14:16:25.427347+0000","last_peered":"2024-09-13T14:16:25.427347+0000","last_clean":"2024-09-13T14:16:25.427347+0000","last_became_active":"2024-09-13T14:15:26.555311+0000","last_became_peered":"2024-09-13T14:15:26.555311+0000","last_unstale":"2024-09-13T14:16:25.427347+0000","last_undegraded":"2024-09-13T14:16:25.427347+0000","last_fullsized":"2024-09-13T14:16:25.427347+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404742+0000","last_change":"2024-09-13T14:15:26.572204+0000","last_active":"2024-09-13T14:16:25.404742+0000","last_peered":"2024-09-13T14:16:25.404742+0000","last_clean":"2024-09-13T14:16:25.404742+0000","last_became_active":"2024-09-13T14:15:26.571310+0000","last_became_peered":"2024-09-13T14:15:26.571310+0000","last_unstale":"2024-09-13T14:16:25.404742+0000","last_undegraded":"2024-09-13T14:16:25.404742+0000","last_fullsized":"2024-09-13T14:16:25.404742+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426113+0000","last_change":"2024-09-13T14:15:26.566676+0000","last_active":"2024-09-13T14:16:25.426113+0000","last_peered":"2024-09-13T14:16:25.426113+0000","last_clean":"2024-09-13T14:16:25.426113+0000","last_became_active":"2024-09-13T14:15:26.566403+0000","last_became_peered":"2024-09-13T14:15:26.566403+0000","last_unstale":"2024-09-13T14:16:25.426113+0000","last_undegraded":"2024-09-13T14:16:25.426113+0000","last_fullsized":"2024-09-13T14:16:25.426113+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399106+0000","last_change":"2024-09-13T14:15:26.566663+0000","last_active":"2024-09-13T14:16:25.399106+0000","last_peered":"2024-09-13T14:16:25.399106+0000","last_clean":"2024-09-13T14:16:25.399106+0000","last_became_active":"2024-09-13T14:15:26.566400+0000","last_became_peered":"2024-09-13T14:15:26.566400+0000","last_unstale":"2024-09-13T14:16:25.399106+0000","last_undegraded":"2024-09-13T14:16:25.399106+0000","last_fullsized":"2024-09-13T14:16:25.399106+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427605+0000","last_change":"2024-09-13T14:15:26.570338+0000","last_active":"2024-09-13T14:16:25.427605+0000","last_peered":"2024-09-13T14:16:25.427605+0000","last_clean":"2024-09-13T14:16:25.427605+0000","last_became_active":"2024-09-13T14:15:26.569776+0000","last_became_peered":"2024-09-13T14:15:26.569776+0000","last_unstale":"2024-09-13T14:16:25.427605+0000","last_undegraded":"2024-09-13T14:16:25.427605+0000","last_fullsized":"2024-09-13T14:16:25.427605+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427114+0000","last_change":"2024-09-13T14:15:26.564796+0000","last_active":"2024-09-13T14:16:25.427114+0000","last_peered":"2024-09-13T14:16:25.427114+0000","last_clean":"2024-09-13T14:16:25.427114+0000","last_became_active":"2024-09-13T14:15:26.563959+0000","last_became_peered":"2024-09-13T14:15:26.563959+0000","last_unstale":"2024-09-13T14:16:25.427114+0000","last_undegraded":"2024-09-13T14:16:25.427114+0000","last_fullsized":"2024-09-13T14:16:25.427114+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426507+0000","last_change":"2024-09-13T14:15:26.567040+0000","last_active":"2024-09-13T14:16:25.426507+0000","last_peered":"2024-09-13T14:16:25.426507+0000","last_clean":"2024-09-13T14:16:25.426507+0000","last_became_active":"2024-09-13T14:15:26.566726+0000","last_became_peered":"2024-09-13T14:15:26.566726+0000","last_unstale":"2024-09-13T14:16:25.426507+0000","last_undegraded":"2024-09-13T14:16:25.426507+0000","last_fullsized":"2024-09-13T14:16:25.426507+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398688+0000","last_change":"2024-09-13T14:15:26.563679+0000","last_active":"2024-09-13T14:16:25.398688+0000","last_peered":"2024-09-13T14:16:25.398688+0000","last_clean":"2024-09-13T14:16:25.398688+0000","last_became_active":"2024-09-13T14:15:26.563402+0000","last_became_peered":"2024-09-13T14:15:26.563402+0000","last_unstale":"2024-09-13T14:16:25.398688+0000","last_undegraded":"2024-09-13T14:16:25.398688+0000","last_fullsized":"2024-09-13T14:16:25.398688+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399996+0000","last_change":"2024-09-13T14:15:26.552634+0000","last_active":"2024-09-13T14:16:25.399996+0000","last_peered":"2024-09-13T14:16:25.399996+0000","last_clean":"2024-09-13T14:16:25.399996+0000","last_became_active":"2024-09-13T14:15:26.552354+0000","last_became_peered":"2024-09-13T14:15:26.552354+0000","last_unstale":"2024-09-13T14:16:25.399996+0000","last_undegraded":"2024-09-13T14:16:25.399996+0000","last_fullsized":"2024-09-13T14:16:25.399996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426019+0000","last_change":"2024-09-13T14:15:26.576136+0000","last_active":"2024-09-13T14:16:25.426019+0000","last_peered":"2024-09-13T14:16:25.426019+0000","last_clean":"2024-09-13T14:16:25.426019+0000","last_became_active":"2024-09-13T14:15:26.575737+0000","last_became_peered":"2024-09-13T14:15:26.575737+0000","last_unstale":"2024-09-13T14:16:25.426019+0000","last_undegraded":"2024-09-13T14:16:25.426019+0000","last_fullsized":"2024-09-13T14:16:25.426019+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402570+0000","last_change":"2024-09-13T14:15:26.566237+0000","last_active":"2024-09-13T14:16:25.402570+0000","last_peered":"2024-09-13T14:16:25.402570+0000","last_clean":"2024-09-13T14:16:25.402570+0000","last_became_active":"2024-09-13T14:15:26.565645+0000","last_became_peered":"2024-09-13T14:15:26.565645+0000","last_unstale":"2024-09-13T14:16:25.402570+0000","last_undegraded":"2024-09-13T14:16:25.402570+0000","last_fullsized":"2024-09-13T14:16:25.402570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404637+0000","last_change":"2024-09-13T14:15:46.965107+0000","last_active":"2024-09-13T14:16:25.404637+0000","last_peered":"2024-09-13T14:16:25.404637+0000","last_clean":"2024-09-13T14:16:25.404637+0000","last_became_active":"2024-09-13T14:15:46.964708+0000","last_became_peered":"2024-09-13T14:15:46.964708+0000","last_unstale":"2024-09-13T14:16:25.404637+0000","last_undegraded":"2024-09-13T14:16:25.404637+0000","last_fullsized":"2024-09-13T14:16:25.404637+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405223+0000","last_change":"2024-09-13T14:15:26.565535+0000","last_active":"2024-09-13T14:16:25.405223+0000","last_peered":"2024-09-13T14:16:25.405223+0000","last_clean":"2024-09-13T14:16:25.405223+0000","last_became_active":"2024-09-13T14:15:26.565091+0000","last_became_peered":"2024-09-13T14:15:26.565091+0000","last_unstale":"2024-09-13T14:16:25.405223+0000","last_undegraded":"2024-09-13T14:16:25.405223+0000","last_fullsized":"2024-09-13T14:16:25.405223+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398039+0000","last_change":"2024-09-13T14:15:26.556686+0000","last_active":"2024-09-13T14:16:25.398039+0000","last_peered":"2024-09-13T14:16:25.398039+0000","last_clean":"2024-09-13T14:16:25.398039+0000","last_became_active":"2024-09-13T14:15:26.556349+0000","last_became_peered":"2024-09-13T14:15:26.556349+0000","last_unstale":"2024-09-13T14:16:25.398039+0000","last_undegraded":"2024-09-13T14:16:25.398039+0000","last_fullsized":"2024-09-13T14:16:25.398039+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427100+0000","last_change":"2024-09-13T14:15:26.564825+0000","last_active":"2024-09-13T14:16:25.427100+0000","last_peered":"2024-09-13T14:16:25.427100+0000","last_clean":"2024-09-13T14:16:25.427100+0000","last_became_active":"2024-09-13T14:15:26.563703+0000","last_became_peered":"2024-09-13T14:15:26.563703+0000","last_unstale":"2024-09-13T14:16:25.427100+0000","last_undegraded":"2024-09-13T14:16:25.427100+0000","last_fullsized":"2024-09-13T14:16:25.427100+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426232+0000","last_change":"2024-09-13T14:15:26.568327+0000","last_active":"2024-09-13T14:16:25.426232+0000","last_peered":"2024-09-13T14:16:25.426232+0000","last_clean":"2024-09-13T14:16:25.426232+0000","last_became_active":"2024-09-13T14:15:26.568070+0000","last_became_peered":"2024-09-13T14:15:26.568070+0000","last_unstale":"2024-09-13T14:16:25.426232+0000","last_undegraded":"2024-09-13T14:16:25.426232+0000","last_fullsized":"2024-09-13T14:16:25.426232+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426139+0000","last_change":"2024-09-13T14:15:26.568339+0000","last_active":"2024-09-13T14:16:25.426139+0000","last_peered":"2024-09-13T14:16:25.426139+0000","last_clean":"2024-09-13T14:16:25.426139+0000","last_became_active":"2024-09-13T14:15:26.568056+0000","last_became_peered":"2024-09-13T14:15:26.568056+0000","last_unstale":"2024-09-13T14:16:25.426139+0000","last_undegraded":"2024-09-13T14:16:25.426139+0000","last_fullsized":"2024-09-13T14:16:25.426139+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404854+0000","last_change":"2024-09-13T14:15:26.572417+0000","last_active":"2024-09-13T14:16:25.404854+0000","last_peered":"2024-09-13T14:16:25.404854+0000","last_clean":"2024-09-13T14:16:25.404854+0000","last_became_active":"2024-09-13T14:15:26.572126+0000","last_became_peered":"2024-09-13T14:15:26.572126+0000","last_unstale":"2024-09-13T14:16:25.404854+0000","last_undegraded":"2024-09-13T14:16:25.404854+0000","last_fullsized":"2024-09-13T14:16:25.404854+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398245+0000","last_change":"2024-09-13T14:15:26.555819+0000","last_active":"2024-09-13T14:16:25.398245+0000","last_peered":"2024-09-13T14:16:25.398245+0000","last_clean":"2024-09-13T14:16:25.398245+0000","last_became_active":"2024-09-13T14:15:26.555172+0000","last_became_peered":"2024-09-13T14:15:26.555172+0000","last_unstale":"2024-09-13T14:16:25.398245+0000","last_undegraded":"2024-09-13T14:16:25.398245+0000","last_fullsized":"2024-09-13T14:16:25.398245+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404692+0000","last_change":"2024-09-13T14:15:26.561070+0000","last_active":"2024-09-13T14:16:25.404692+0000","last_peered":"2024-09-13T14:16:25.404692+0000","last_clean":"2024-09-13T14:16:25.404692+0000","last_became_active":"2024-09-13T14:15:26.559771+0000","last_became_peered":"2024-09-13T14:15:26.559771+0000","last_unstale":"2024-09-13T14:16:25.404692+0000","last_undegraded":"2024-09-13T14:16:25.404692+0000","last_fullsized":"2024-09-13T14:16:25.404692+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399852+0000","last_change":"2024-09-13T14:15:26.554841+0000","last_active":"2024-09-13T14:16:25.399852+0000","last_peered":"2024-09-13T14:16:25.399852+0000","last_clean":"2024-09-13T14:16:25.399852+0000","last_became_active":"2024-09-13T14:15:26.553231+0000","last_became_peered":"2024-09-13T14:15:26.553231+0000","last_unstale":"2024-09-13T14:16:25.399852+0000","last_undegraded":"2024-09-13T14:16:25.399852+0000","last_fullsized":"2024-09-13T14:16:25.399852+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427182+0000","last_change":"2024-09-13T14:15:26.551060+0000","last_active":"2024-09-13T14:16:25.427182+0000","last_peered":"2024-09-13T14:16:25.427182+0000","last_clean":"2024-09-13T14:16:25.427182+0000","last_became_active":"2024-09-13T14:15:26.550739+0000","last_became_peered":"2024-09-13T14:15:26.550739+0000","last_unstale":"2024-09-13T14:16:25.427182+0000","last_undegraded":"2024-09-13T14:16:25.427182+0000","last_fullsized":"2024-09-13T14:16:25.427182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402995+0000","last_change":"2024-09-13T14:15:26.559670+0000","last_active":"2024-09-13T14:16:25.402995+0000","last_peered":"2024-09-13T14:16:25.402995+0000","last_clean":"2024-09-13T14:16:25.402995+0000","last_became_active":"2024-09-13T14:15:26.559401+0000","last_became_peered":"2024-09-13T14:15:26.559401+0000","last_unstale":"2024-09-13T14:16:25.402995+0000","last_undegraded":"2024-09-13T14:16:25.402995+0000","last_fullsized":"2024-09-13T14:16:25.402995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399213+0000","last_change":"2024-09-13T14:15:26.570832+0000","last_active":"2024-09-13T14:16:25.399213+0000","last_peered":"2024-09-13T14:16:25.399213+0000","last_clean":"2024-09-13T14:16:25.399213+0000","last_became_active":"2024-09-13T14:15:26.570531+0000","last_became_peered":"2024-09-13T14:15:26.570531+0000","last_unstale":"2024-09-13T14:16:25.399213+0000","last_undegraded":"2024-09-13T14:16:25.399213+0000","last_fullsized":"2024-09-13T14:16:25.399213+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426207+0000","last_change":"2024-09-13T14:15:26.573053+0000","last_active":"2024-09-13T14:16:25.426207+0000","last_peered":"2024-09-13T14:16:25.426207+0000","last_clean":"2024-09-13T14:16:25.426207+0000","last_became_active":"2024-09-13T14:15:26.572783+0000","last_became_peered":"2024-09-13T14:15:26.572783+0000","last_unstale":"2024-09-13T14:16:25.426207+0000","last_undegraded":"2024-09-13T14:16:25.426207+0000","last_fullsized":"2024-09-13T14:16:25.426207+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404968+0000","last_change":"2024-09-13T14:15:26.569346+0000","last_active":"2024-09-13T14:16:25.404968+0000","last_peered":"2024-09-13T14:16:25.404968+0000","last_clean":"2024-09-13T14:16:25.404968+0000","last_became_active":"2024-09-13T14:15:26.568939+0000","last_became_peered":"2024-09-13T14:15:26.568939+0000","last_unstale":"2024-09-13T14:16:25.404968+0000","last_undegraded":"2024-09-13T14:16:25.404968+0000","last_fullsized":"2024-09-13T14:16:25.404968+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397771+0000","last_change":"2024-09-13T14:15:26.562543+0000","last_active":"2024-09-13T14:16:25.397771+0000","last_peered":"2024-09-13T14:16:25.397771+0000","last_clean":"2024-09-13T14:16:25.397771+0000","last_became_active":"2024-09-13T14:15:26.562267+0000","last_became_peered":"2024-09-13T14:15:26.562267+0000","last_unstale":"2024-09-13T14:16:25.397771+0000","last_undegraded":"2024-09-13T14:16:25.397771+0000","last_fullsized":"2024-09-13T14:16:25.397771+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425659+0000","last_change":"2024-09-13T14:15:26.568440+0000","last_active":"2024-09-13T14:16:25.425659+0000","last_peered":"2024-09-13T14:16:25.425659+0000","last_clean":"2024-09-13T14:16:25.425659+0000","last_became_active":"2024-09-13T14:15:26.568169+0000","last_became_peered":"2024-09-13T14:15:26.568169+0000","last_unstale":"2024-09-13T14:16:25.425659+0000","last_undegraded":"2024-09-13T14:16:25.425659+0000","last_fullsized":"2024-09-13T14:16:25.425659+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398398+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:16:25.398398+0000","last_peered":"2024-09-13T14:16:25.398398+0000","last_clean":"2024-09-13T14:16:25.398398+0000","last_became_active":"2024-09-13T14:15:46.963029+0000","last_became_peered":"2024-09-13T14:15:46.963029+0000","last_unstale":"2024-09-13T14:16:25.398398+0000","last_undegraded":"2024-09-13T14:16:25.398398+0000","last_fullsized":"2024-09-13T14:16:25.398398+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400198+0000","last_change":"2024-09-13T14:15:22.541205+0000","last_active":"2024-09-13T14:16:25.400198+0000","last_peered":"2024-09-13T14:16:25.400198+0000","last_clean":"2024-09-13T14:16:25.400198+0000","last_became_active":"2024-09-13T14:15:22.539574+0000","last_became_peered":"2024-09-13T14:15:22.539574+0000","last_unstale":"2024-09-13T14:16:25.400198+0000","last_undegraded":"2024-09-13T14:16:25.400198+0000","last_fullsized":"2024-09-13T14:16:25.400198+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397655+0000","last_change":"2024-09-13T14:15:46.606065+0000","last_active":"2024-09-13T14:16:25.397655+0000","last_peered":"2024-09-13T14:16:25.397655+0000","last_clean":"2024-09-13T14:16:25.397655+0000","last_became_active":"2024-09-13T14:15:46.605583+0000","last_became_peered":"2024-09-13T14:15:46.605583+0000","last_unstale":"2024-09-13T14:16:25.397655+0000","last_undegraded":"2024-09-13T14:16:25.397655+0000","last_fullsized":"2024-09-13T14:16:25.397655+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:21.907855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425880+0000","last_change":"2024-09-13T14:15:22.543444+0000","last_active":"2024-09-13T14:16:25.425880+0000","last_peered":"2024-09-13T14:16:25.425880+0000","last_clean":"2024-09-13T14:16:25.425880+0000","last_became_active":"2024-09-13T14:15:22.543138+0000","last_became_peered":"2024-09-13T14:15:22.543138+0000","last_unstale":"2024-09-13T14:16:25.425880+0000","last_undegraded":"2024-09-13T14:16:25.425880+0000","last_fullsized":"2024-09-13T14:16:25.425880+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399765+0000","last_change":"2024-09-13T14:15:22.541457+0000","last_active":"2024-09-13T14:16:25.399765+0000","last_peered":"2024-09-13T14:16:25.399765+0000","last_clean":"2024-09-13T14:16:25.399765+0000","last_became_active":"2024-09-13T14:15:22.539753+0000","last_became_peered":"2024-09-13T14:15:22.539753+0000","last_unstale":"2024-09-13T14:16:25.399765+0000","last_undegraded":"2024-09-13T14:16:25.399765+0000","last_fullsized":"2024-09-13T14:16:25.399765+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399167+0000","last_change":"2024-09-13T14:15:26.571137+0000","last_active":"2024-09-13T14:16:25.399167+0000","last_peered":"2024-09-13T14:16:25.399167+0000","last_clean":"2024-09-13T14:16:25.399167+0000","last_became_active":"2024-09-13T14:15:26.570884+0000","last_became_peered":"2024-09-13T14:15:26.570884+0000","last_unstale":"2024-09-13T14:16:25.399167+0000","last_undegraded":"2024-09-13T14:16:25.399167+0000","last_fullsized":"2024-09-13T14:16:25.399167+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404803+0000","last_change":"2024-09-13T14:15:22.546875+0000","last_active":"2024-09-13T14:16:25.404803+0000","last_peered":"2024-09-13T14:16:25.404803+0000","last_clean":"2024-09-13T14:16:25.404803+0000","last_became_active":"2024-09-13T14:15:22.545863+0000","last_became_peered":"2024-09-13T14:15:22.545863+0000","last_unstale":"2024-09-13T14:16:25.404803+0000","last_undegraded":"2024-09-13T14:16:25.404803+0000","last_fullsized":"2024-09-13T14:16:25.404803+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402439+0000","last_change":"2024-09-13T14:15:26.560575+0000","last_active":"2024-09-13T14:16:25.402439+0000","last_peered":"2024-09-13T14:16:25.402439+0000","last_clean":"2024-09-13T14:16:25.402439+0000","last_became_active":"2024-09-13T14:15:26.560291+0000","last_became_peered":"2024-09-13T14:15:26.560291+0000","last_unstale":"2024-09-13T14:16:25.402439+0000","last_undegraded":"2024-09-13T14:16:25.402439+0000","last_fullsized":"2024-09-13T14:16:25.402439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427015+0000","last_change":"2024-09-13T14:15:22.546137+0000","last_active":"2024-09-13T14:16:25.427015+0000","last_peered":"2024-09-13T14:16:25.427015+0000","last_clean":"2024-09-13T14:16:25.427015+0000","last_became_active":"2024-09-13T14:15:22.545833+0000","last_became_peered":"2024-09-13T14:15:22.545833+0000","last_unstale":"2024-09-13T14:16:25.427015+0000","last_undegraded":"2024-09-13T14:16:25.427015+0000","last_fullsized":"2024-09-13T14:16:25.427015+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404982+0000","last_change":"2024-09-13T14:15:26.566451+0000","last_active":"2024-09-13T14:16:25.404982+0000","last_peered":"2024-09-13T14:16:25.404982+0000","last_clean":"2024-09-13T14:16:25.404982+0000","last_became_active":"2024-09-13T14:15:26.565363+0000","last_became_peered":"2024-09-13T14:15:26.565363+0000","last_unstale":"2024-09-13T14:16:25.404982+0000","last_undegraded":"2024-09-13T14:16:25.404982+0000","last_fullsized":"2024-09-13T14:16:25.404982+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405218+0000","last_change":"2024-09-13T14:15:22.540966+0000","last_active":"2024-09-13T14:16:25.405218+0000","last_peered":"2024-09-13T14:16:25.405218+0000","last_clean":"2024-09-13T14:16:25.405218+0000","last_became_active":"2024-09-13T14:15:22.540611+0000","last_became_peered":"2024-09-13T14:15:22.540611+0000","last_unstale":"2024-09-13T14:16:25.405218+0000","last_undegraded":"2024-09-13T14:16:25.405218+0000","last_fullsized":"2024-09-13T14:16:25.405218+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426404+0000","last_change":"2024-09-13T14:15:26.573426+0000","last_active":"2024-09-13T14:16:25.426404+0000","last_peered":"2024-09-13T14:16:25.426404+0000","last_clean":"2024-09-13T14:16:25.426404+0000","last_became_active":"2024-09-13T14:15:26.573138+0000","last_became_peered":"2024-09-13T14:15:26.573138+0000","last_unstale":"2024-09-13T14:16:25.426404+0000","last_undegraded":"2024-09-13T14:16:25.426404+0000","last_fullsized":"2024-09-13T14:16:25.426404+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403094+0000","last_change":"2024-09-13T14:15:22.543031+0000","last_active":"2024-09-13T14:16:25.403094+0000","last_peered":"2024-09-13T14:16:25.403094+0000","last_clean":"2024-09-13T14:16:25.403094+0000","last_became_active":"2024-09-13T14:15:22.542739+0000","last_became_peered":"2024-09-13T14:15:22.542739+0000","last_unstale":"2024-09-13T14:16:25.403094+0000","last_undegraded":"2024-09-13T14:16:25.403094+0000","last_fullsized":"2024-09-13T14:16:25.403094+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398640+0000","last_change":"2024-09-13T14:15:26.551575+0000","last_active":"2024-09-13T14:16:25.398640+0000","last_peered":"2024-09-13T14:16:25.398640+0000","last_clean":"2024-09-13T14:16:25.398640+0000","last_became_active":"2024-09-13T14:15:26.550913+0000","last_became_peered":"2024-09-13T14:15:26.550913+0000","last_unstale":"2024-09-13T14:16:25.398640+0000","last_undegraded":"2024-09-13T14:16:25.398640+0000","last_fullsized":"2024-09-13T14:16:25.398640+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426327+0000","last_change":"2024-09-13T14:15:22.539191+0000","last_active":"2024-09-13T14:16:25.426327+0000","last_peered":"2024-09-13T14:16:25.426327+0000","last_clean":"2024-09-13T14:16:25.426327+0000","last_became_active":"2024-09-13T14:15:22.538864+0000","last_became_peered":"2024-09-13T14:15:22.538864+0000","last_unstale":"2024-09-13T14:16:25.426327+0000","last_undegraded":"2024-09-13T14:16:25.426327+0000","last_fullsized":"2024-09-13T14:16:25.426327+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426673+0000","last_change":"2024-09-13T14:15:26.567770+0000","last_active":"2024-09-13T14:16:25.426673+0000","last_peered":"2024-09-13T14:16:25.426673+0000","last_clean":"2024-09-13T14:16:25.426673+0000","last_became_active":"2024-09-13T14:15:26.567513+0000","last_became_peered":"2024-09-13T14:15:26.567513+0000","last_unstale":"2024-09-13T14:16:25.426673+0000","last_undegraded":"2024-09-13T14:16:25.426673+0000","last_fullsized":"2024-09-13T14:16:25.426673+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426237+0000","last_change":"2024-09-13T14:15:22.545520+0000","last_active":"2024-09-13T14:16:25.426237+0000","last_peered":"2024-09-13T14:16:25.426237+0000","last_clean":"2024-09-13T14:16:25.426237+0000","last_became_active":"2024-09-13T14:15:22.545135+0000","last_became_peered":"2024-09-13T14:15:22.545135+0000","last_unstale":"2024-09-13T14:16:25.426237+0000","last_undegraded":"2024-09-13T14:16:25.426237+0000","last_fullsized":"2024-09-13T14:16:25.426237+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426065+0000","last_change":"2024-09-13T14:15:26.575023+0000","last_active":"2024-09-13T14:16:25.426065+0000","last_peered":"2024-09-13T14:16:25.426065+0000","last_clean":"2024-09-13T14:16:25.426065+0000","last_became_active":"2024-09-13T14:15:26.574648+0000","last_became_peered":"2024-09-13T14:15:26.574648+0000","last_unstale":"2024-09-13T14:16:25.426065+0000","last_undegraded":"2024-09-13T14:16:25.426065+0000","last_fullsized":"2024-09-13T14:16:25.426065+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405021+0000","last_change":"2024-09-13T14:15:22.546988+0000","last_active":"2024-09-13T14:16:25.405021+0000","last_peered":"2024-09-13T14:16:25.405021+0000","last_clean":"2024-09-13T14:16:25.405021+0000","last_became_active":"2024-09-13T14:15:22.545255+0000","last_became_peered":"2024-09-13T14:15:22.545255+0000","last_unstale":"2024-09-13T14:16:25.405021+0000","last_undegraded":"2024-09-13T14:16:25.405021+0000","last_fullsized":"2024-09-13T14:16:25.405021+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426955+0000","last_change":"2024-09-13T14:15:26.555682+0000","last_active":"2024-09-13T14:16:25.426955+0000","last_peered":"2024-09-13T14:16:25.426955+0000","last_clean":"2024-09-13T14:16:25.426955+0000","last_became_active":"2024-09-13T14:15:26.555391+0000","last_became_peered":"2024-09-13T14:15:26.555391+0000","last_unstale":"2024-09-13T14:16:25.426955+0000","last_undegraded":"2024-09-13T14:16:25.426955+0000","last_fullsized":"2024-09-13T14:16:25.426955+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425985+0000","last_change":"2024-09-13T14:15:22.542050+0000","last_active":"2024-09-13T14:16:25.425985+0000","last_peered":"2024-09-13T14:16:25.425985+0000","last_clean":"2024-09-13T14:16:25.425985+0000","last_became_active":"2024-09-13T14:15:22.541726+0000","last_became_peered":"2024-09-13T14:15:22.541726+0000","last_unstale":"2024-09-13T14:16:25.425985+0000","last_undegraded":"2024-09-13T14:16:25.425985+0000","last_fullsized":"2024-09-13T14:16:25.425985+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425693+0000","last_change":"2024-09-13T14:15:26.567258+0000","last_active":"2024-09-13T14:16:25.425693+0000","last_peered":"2024-09-13T14:16:25.425693+0000","last_clean":"2024-09-13T14:16:25.425693+0000","last_became_active":"2024-09-13T14:15:26.566924+0000","last_became_peered":"2024-09-13T14:15:26.566924+0000","last_unstale":"2024-09-13T14:16:25.425693+0000","last_undegraded":"2024-09-13T14:16:25.425693+0000","last_fullsized":"2024-09-13T14:16:25.425693+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397636+0000","last_change":"2024-09-13T14:15:22.541058+0000","last_active":"2024-09-13T14:16:25.397636+0000","last_peered":"2024-09-13T14:16:25.397636+0000","last_clean":"2024-09-13T14:16:25.397636+0000","last_became_active":"2024-09-13T14:15:22.540793+0000","last_became_peered":"2024-09-13T14:15:22.540793+0000","last_unstale":"2024-09-13T14:16:25.397636+0000","last_undegraded":"2024-09-13T14:16:25.397636+0000","last_fullsized":"2024-09-13T14:16:25.397636+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426226+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:16:25.426226+0000","last_peered":"2024-09-13T14:16:25.426226+0000","last_clean":"2024-09-13T14:16:25.426226+0000","last_became_active":"2024-09-13T14:15:26.562669+0000","last_became_peered":"2024-09-13T14:15:26.562669+0000","last_unstale":"2024-09-13T14:16:25.426226+0000","last_undegraded":"2024-09-13T14:16:25.426226+0000","last_fullsized":"2024-09-13T14:16:25.426226+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398941+0000","last_change":"2024-09-13T14:15:22.540681+0000","last_active":"2024-09-13T14:16:25.398941+0000","last_peered":"2024-09-13T14:16:25.398941+0000","last_clean":"2024-09-13T14:16:25.398941+0000","last_became_active":"2024-09-13T14:15:22.540386+0000","last_became_peered":"2024-09-13T14:15:22.540386+0000","last_unstale":"2024-09-13T14:16:25.398941+0000","last_undegraded":"2024-09-13T14:16:25.398941+0000","last_fullsized":"2024-09-13T14:16:25.398941+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426867+0000","last_change":"2024-09-13T14:15:26.559281+0000","last_active":"2024-09-13T14:16:25.426867+0000","last_peered":"2024-09-13T14:16:25.426867+0000","last_clean":"2024-09-13T14:16:25.426867+0000","last_became_active":"2024-09-13T14:15:26.558987+0000","last_became_peered":"2024-09-13T14:15:26.558987+0000","last_unstale":"2024-09-13T14:16:25.426867+0000","last_undegraded":"2024-09-13T14:16:25.426867+0000","last_fullsized":"2024-09-13T14:16:25.426867+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398225+0000","last_change":"2024-09-13T14:15:22.544147+0000","last_active":"2024-09-13T14:16:25.398225+0000","last_peered":"2024-09-13T14:16:25.398225+0000","last_clean":"2024-09-13T14:16:25.398225+0000","last_became_active":"2024-09-13T14:15:22.543782+0000","last_became_peered":"2024-09-13T14:15:22.543782+0000","last_unstale":"2024-09-13T14:16:25.398225+0000","last_undegraded":"2024-09-13T14:16:25.398225+0000","last_fullsized":"2024-09-13T14:16:25.398225+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427487+0000","last_change":"2024-09-13T14:15:26.570603+0000","last_active":"2024-09-13T14:16:25.427487+0000","last_peered":"2024-09-13T14:16:25.427487+0000","last_clean":"2024-09-13T14:16:25.427487+0000","last_became_active":"2024-09-13T14:15:26.569357+0000","last_became_peered":"2024-09-13T14:15:26.569357+0000","last_unstale":"2024-09-13T14:16:25.427487+0000","last_undegraded":"2024-09-13T14:16:25.427487+0000","last_fullsized":"2024-09-13T14:16:25.427487+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403205+0000","last_change":"2024-09-13T14:15:22.546363+0000","last_active":"2024-09-13T14:16:25.403205+0000","last_peered":"2024-09-13T14:16:25.403205+0000","last_clean":"2024-09-13T14:16:25.403205+0000","last_became_active":"2024-09-13T14:15:22.544417+0000","last_became_peered":"2024-09-13T14:15:22.544417+0000","last_unstale":"2024-09-13T14:16:25.403205+0000","last_undegraded":"2024-09-13T14:16:25.403205+0000","last_fullsized":"2024-09-13T14:16:25.403205+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404958+0000","last_change":"2024-09-13T14:15:26.572445+0000","last_active":"2024-09-13T14:16:25.404958+0000","last_peered":"2024-09-13T14:16:25.404958+0000","last_clean":"2024-09-13T14:16:25.404958+0000","last_became_active":"2024-09-13T14:15:26.572166+0000","last_became_peered":"2024-09-13T14:15:26.572166+0000","last_unstale":"2024-09-13T14:16:25.404958+0000","last_undegraded":"2024-09-13T14:16:25.404958+0000","last_fullsized":"2024-09-13T14:16:25.404958+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398707+0000","last_change":"2024-09-13T14:15:22.542920+0000","last_active":"2024-09-13T14:16:25.398707+0000","last_peered":"2024-09-13T14:16:25.398707+0000","last_clean":"2024-09-13T14:16:25.398707+0000","last_became_active":"2024-09-13T14:15:22.542641+0000","last_became_peered":"2024-09-13T14:15:22.542641+0000","last_unstale":"2024-09-13T14:16:25.398707+0000","last_undegraded":"2024-09-13T14:16:25.398707+0000","last_fullsized":"2024-09-13T14:16:25.398707+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426854+0000","last_change":"2024-09-13T14:15:26.572195+0000","last_active":"2024-09-13T14:16:25.426854+0000","last_peered":"2024-09-13T14:16:25.426854+0000","last_clean":"2024-09-13T14:16:25.426854+0000","last_became_active":"2024-09-13T14:15:26.571003+0000","last_became_peered":"2024-09-13T14:15:26.571003+0000","last_unstale":"2024-09-13T14:16:25.426854+0000","last_undegraded":"2024-09-13T14:16:25.426854+0000","last_fullsized":"2024-09-13T14:16:25.426854+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:33.691439+0000","last_change":"2024-09-13T14:16:25.426886+0000","last_active":"2024-09-13T14:16:33.691439+0000","last_peered":"2024-09-13T14:16:33.691439+0000","last_clean":"2024-09-13T14:16:33.691439+0000","last_became_active":"2024-09-13T14:16:21.376845+0000","last_became_peered":"2024-09-13T14:16:21.376845+0000","last_unstale":"2024-09-13T14:16:33.691439+0000","last_undegraded":"2024-09-13T14:16:33.691439+0000","last_fullsized":"2024-09-13T14:16:33.691439+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:10.386795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055469799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"60'79","reported_seq":57,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427505+0000","last_change":"2024-09-13T14:14:57.479980+0000","last_active":"2024-09-13T14:16:25.427505+0000","last_peered":"2024-09-13T14:16:25.427505+0000","last_clean":"2024-09-13T14:16:25.427505+0000","last_became_active":"2024-09-13T14:14:57.475298+0000","last_became_peered":"2024-09-13T14:14:57.475298+0000","last_unstale":"2024-09-13T14:16:25.427505+0000","last_undegraded":"2024-09-13T14:16:25.427505+0000","last_fullsized":"2024-09-13T14:16:25.427505+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_clean_scrub_stamp":"2024-09-13T14:12:56.059430+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.404852+0000","last_change":"2024-09-13T14:15:22.542042+0000","last_active":"2024-09-13T14:16:25.404852+0000","last_peered":"2024-09-13T14:16:25.404852+0000","last_clean":"2024-09-13T14:16:25.404852+0000","last_became_active":"2024-09-13T14:15:22.541756+0000","last_became_peered":"2024-09-13T14:15:22.541756+0000","last_unstale":"2024-09-13T14:16:25.404852+0000","last_undegraded":"2024-09-13T14:16:25.404852+0000","last_fullsized":"2024-09-13T14:16:25.404852+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398634+0000","last_change":"2024-09-13T14:15:26.565510+0000","last_active":"2024-09-13T14:16:25.398634+0000","last_peered":"2024-09-13T14:16:25.398634+0000","last_clean":"2024-09-13T14:16:25.398634+0000","last_became_active":"2024-09-13T14:15:26.565213+0000","last_became_peered":"2024-09-13T14:15:26.565213+0000","last_unstale":"2024-09-13T14:16:25.398634+0000","last_undegraded":"2024-09-13T14:16:25.398634+0000","last_fullsized":"2024-09-13T14:16:25.398634+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.002691+0000","last_change":"2024-09-13T14:16:25.426290+0000","last_active":"2024-09-13T14:16:34.002691+0000","last_peered":"2024-09-13T14:16:34.002691+0000","last_clean":"2024-09-13T14:16:34.002691+0000","last_became_active":"2024-09-13T14:16:22.202033+0000","last_became_peered":"2024-09-13T14:16:22.202033+0000","last_unstale":"2024-09-13T14:16:34.002691+0000","last_undegraded":"2024-09-13T14:16:34.002691+0000","last_fullsized":"2024-09-13T14:16:34.002691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:48.139777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071034400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427357+0000","last_change":"2024-09-13T14:15:22.550379+0000","last_active":"2024-09-13T14:16:25.427357+0000","last_peered":"2024-09-13T14:16:25.427357+0000","last_clean":"2024-09-13T14:16:25.427357+0000","last_became_active":"2024-09-13T14:15:22.549995+0000","last_became_peered":"2024-09-13T14:15:22.549995+0000","last_unstale":"2024-09-13T14:16:25.427357+0000","last_undegraded":"2024-09-13T14:16:25.427357+0000","last_fullsized":"2024-09-13T14:16:25.427357+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425608+0000","last_change":"2024-09-13T14:15:46.965011+0000","last_active":"2024-09-13T14:16:25.425608+0000","last_peered":"2024-09-13T14:16:25.425608+0000","last_clean":"2024-09-13T14:16:25.425608+0000","last_became_active":"2024-09-13T14:15:46.964646+0000","last_became_peered":"2024-09-13T14:15:46.964646+0000","last_unstale":"2024-09-13T14:16:25.425608+0000","last_undegraded":"2024-09-13T14:16:25.425608+0000","last_fullsized":"2024-09-13T14:16:25.425608+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:07.136822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.174222+0000","last_change":"2024-09-13T14:16:25.403479+0000","last_active":"2024-09-13T14:16:34.174222+0000","last_peered":"2024-09-13T14:16:34.174222+0000","last_clean":"2024-09-13T14:16:34.174222+0000","last_became_active":"2024-09-13T14:16:21.605160+0000","last_became_peered":"2024-09-13T14:16:21.605160+0000","last_unstale":"2024-09-13T14:16:34.174222+0000","last_undegraded":"2024-09-13T14:16:34.174222+0000","last_fullsized":"2024-09-13T14:16:34.174222+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:48.959952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034207600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398529+0000","last_change":"2024-09-13T14:15:22.544154+0000","last_active":"2024-09-13T14:16:25.398529+0000","last_peered":"2024-09-13T14:16:25.398529+0000","last_clean":"2024-09-13T14:16:25.398529+0000","last_became_active":"2024-09-13T14:15:22.543880+0000","last_became_peered":"2024-09-13T14:15:22.543880+0000","last_unstale":"2024-09-13T14:16:25.398529+0000","last_undegraded":"2024-09-13T14:16:25.398529+0000","last_fullsized":"2024-09-13T14:16:25.398529+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402399+0000","last_change":"2024-09-13T14:15:26.553134+0000","last_active":"2024-09-13T14:16:25.402399+0000","last_peered":"2024-09-13T14:16:25.402399+0000","last_clean":"2024-09-13T14:16:25.402399+0000","last_became_active":"2024-09-13T14:15:26.552054+0000","last_became_peered":"2024-09-13T14:15:26.552054+0000","last_unstale":"2024-09-13T14:16:25.402399+0000","last_undegraded":"2024-09-13T14:16:25.402399+0000","last_fullsized":"2024-09-13T14:16:25.402399+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.173862+0000","last_change":"2024-09-13T14:16:25.403217+0000","last_active":"2024-09-13T14:16:34.173862+0000","last_peered":"2024-09-13T14:16:34.173862+0000","last_clean":"2024-09-13T14:16:34.173862+0000","last_became_active":"2024-09-13T14:16:22.202029+0000","last_became_peered":"2024-09-13T14:16:22.202029+0000","last_unstale":"2024-09-13T14:16:34.173862+0000","last_undegraded":"2024-09-13T14:16:34.173862+0000","last_fullsized":"2024-09-13T14:16:34.173862+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:48.284490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000388303,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405168+0000","last_change":"2024-09-13T14:15:22.545901+0000","last_active":"2024-09-13T14:16:25.405168+0000","last_peered":"2024-09-13T14:16:25.405168+0000","last_clean":"2024-09-13T14:16:25.405168+0000","last_became_active":"2024-09-13T14:15:22.545610+0000","last_became_peered":"2024-09-13T14:15:22.545610+0000","last_unstale":"2024-09-13T14:16:25.405168+0000","last_undegraded":"2024-09-13T14:16:25.405168+0000","last_fullsized":"2024-09-13T14:16:25.405168+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398541+0000","last_change":"2024-09-13T14:15:26.562994+0000","last_active":"2024-09-13T14:16:25.398541+0000","last_peered":"2024-09-13T14:16:25.398541+0000","last_clean":"2024-09-13T14:16:25.398541+0000","last_became_active":"2024-09-13T14:15:26.562679+0000","last_became_peered":"2024-09-13T14:15:26.562679+0000","last_unstale":"2024-09-13T14:16:25.398541+0000","last_undegraded":"2024-09-13T14:16:25.398541+0000","last_fullsized":"2024-09-13T14:16:25.398541+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.173690+0000","last_change":"2024-09-13T14:16:25.403491+0000","last_active":"2024-09-13T14:16:34.173690+0000","last_peered":"2024-09-13T14:16:34.173690+0000","last_clean":"2024-09-13T14:16:34.173690+0000","last_became_active":"2024-09-13T14:16:21.605236+0000","last_became_peered":"2024-09-13T14:16:21.605236+0000","last_unstale":"2024-09-13T14:16:34.173690+0000","last_undegraded":"2024-09-13T14:16:34.173690+0000","last_fullsized":"2024-09-13T14:16:34.173690+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:47.051775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097564099999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403019+0000","last_change":"2024-09-13T14:15:22.549591+0000","last_active":"2024-09-13T14:16:25.403019+0000","last_peered":"2024-09-13T14:16:25.403019+0000","last_clean":"2024-09-13T14:16:25.403019+0000","last_became_active":"2024-09-13T14:15:22.549305+0000","last_became_peered":"2024-09-13T14:15:22.549305+0000","last_unstale":"2024-09-13T14:16:25.403019+0000","last_undegraded":"2024-09-13T14:16:25.403019+0000","last_fullsized":"2024-09-13T14:16:25.403019+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405183+0000","last_change":"2024-09-13T14:15:26.562976+0000","last_active":"2024-09-13T14:16:25.405183+0000","last_peered":"2024-09-13T14:16:25.405183+0000","last_clean":"2024-09-13T14:16:25.405183+0000","last_became_active":"2024-09-13T14:15:26.561391+0000","last_became_peered":"2024-09-13T14:15:26.561391+0000","last_unstale":"2024-09-13T14:16:25.405183+0000","last_undegraded":"2024-09-13T14:16:25.405183+0000","last_fullsized":"2024-09-13T14:16:25.405183+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.002319+0000","last_change":"2024-09-13T14:16:25.426585+0000","last_active":"2024-09-13T14:16:34.002319+0000","last_peered":"2024-09-13T14:16:34.002319+0000","last_clean":"2024-09-13T14:16:34.002319+0000","last_became_active":"2024-09-13T14:16:21.604772+0000","last_became_peered":"2024-09-13T14:16:21.604772+0000","last_unstale":"2024-09-13T14:16:34.002319+0000","last_undegraded":"2024-09-13T14:16:34.002319+0000","last_fullsized":"2024-09-13T14:16:34.002319+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:51.564452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000446272,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427204+0000","last_change":"2024-09-13T14:15:22.550059+0000","last_active":"2024-09-13T14:16:25.427204+0000","last_peered":"2024-09-13T14:16:25.427204+0000","last_clean":"2024-09-13T14:16:25.427204+0000","last_became_active":"2024-09-13T14:15:22.549682+0000","last_became_peered":"2024-09-13T14:15:22.549682+0000","last_unstale":"2024-09-13T14:16:25.427204+0000","last_undegraded":"2024-09-13T14:16:25.427204+0000","last_fullsized":"2024-09-13T14:16:25.427204+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402936+0000","last_change":"2024-09-13T14:15:26.565253+0000","last_active":"2024-09-13T14:16:25.402936+0000","last_peered":"2024-09-13T14:16:25.402936+0000","last_clean":"2024-09-13T14:16:25.402936+0000","last_became_active":"2024-09-13T14:15:26.564972+0000","last_became_peered":"2024-09-13T14:15:26.564972+0000","last_unstale":"2024-09-13T14:16:25.402936+0000","last_undegraded":"2024-09-13T14:16:25.402936+0000","last_fullsized":"2024-09-13T14:16:25.402936+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:33.691085+0000","last_change":"2024-09-13T14:16:25.399510+0000","last_active":"2024-09-13T14:16:33.691085+0000","last_peered":"2024-09-13T14:16:33.691085+0000","last_clean":"2024-09-13T14:16:33.691085+0000","last_became_active":"2024-09-13T14:16:21.374161+0000","last_became_peered":"2024-09-13T14:16:21.374161+0000","last_unstale":"2024-09-13T14:16:33.691085+0000","last_undegraded":"2024-09-13T14:16:33.691085+0000","last_fullsized":"2024-09-13T14:16:33.691085+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:24:25.293852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049448099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.403350+0000","last_change":"2024-09-13T14:15:22.547223+0000","last_active":"2024-09-13T14:16:25.403350+0000","last_peered":"2024-09-13T14:16:25.403350+0000","last_clean":"2024-09-13T14:16:25.403350+0000","last_became_active":"2024-09-13T14:15:22.546934+0000","last_became_peered":"2024-09-13T14:15:22.546934+0000","last_unstale":"2024-09-13T14:16:25.403350+0000","last_undegraded":"2024-09-13T14:16:25.403350+0000","last_fullsized":"2024-09-13T14:16:25.403350+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399968+0000","last_change":"2024-09-13T14:15:26.571004+0000","last_active":"2024-09-13T14:16:25.399968+0000","last_peered":"2024-09-13T14:16:25.399968+0000","last_clean":"2024-09-13T14:16:25.399968+0000","last_became_active":"2024-09-13T14:15:26.570753+0000","last_became_peered":"2024-09-13T14:15:26.570753+0000","last_unstale":"2024-09-13T14:16:25.399968+0000","last_undegraded":"2024-09-13T14:16:25.399968+0000","last_fullsized":"2024-09-13T14:16:25.399968+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:16:34.009686+0000","last_change":"2024-09-13T14:16:25.399066+0000","last_active":"2024-09-13T14:16:34.009686+0000","last_peered":"2024-09-13T14:16:34.009686+0000","last_clean":"2024-09-13T14:16:34.009686+0000","last_became_active":"2024-09-13T14:16:21.605337+0000","last_became_peered":"2024-09-13T14:16:21.605337+0000","last_unstale":"2024-09-13T14:16:34.009686+0000","last_undegraded":"2024-09-13T14:16:34.009686+0000","last_fullsized":"2024-09-13T14:16:34.009686+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:12:20.334474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058535100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427115+0000","last_change":"2024-09-13T14:15:22.545342+0000","last_active":"2024-09-13T14:16:25.427115+0000","last_peered":"2024-09-13T14:16:25.427115+0000","last_clean":"2024-09-13T14:16:25.427115+0000","last_became_active":"2024-09-13T14:15:22.545067+0000","last_became_peered":"2024-09-13T14:15:22.545067+0000","last_unstale":"2024-09-13T14:16:25.427115+0000","last_undegraded":"2024-09-13T14:16:25.427115+0000","last_fullsized":"2024-09-13T14:16:25.427115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.397966+0000","last_change":"2024-09-13T14:15:46.963528+0000","last_active":"2024-09-13T14:16:25.397966+0000","last_peered":"2024-09-13T14:16:25.397966+0000","last_clean":"2024-09-13T14:16:25.397966+0000","last_became_active":"2024-09-13T14:15:46.963216+0000","last_became_peered":"2024-09-13T14:15:46.963216+0000","last_unstale":"2024-09-13T14:16:25.397966+0000","last_undegraded":"2024-09-13T14:16:25.397966+0000","last_fullsized":"2024-09-13T14:16:25.397966+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:29.430917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427235+0000","last_change":"2024-09-13T14:15:22.550293+0000","last_active":"2024-09-13T14:16:25.427235+0000","last_peered":"2024-09-13T14:16:25.427235+0000","last_clean":"2024-09-13T14:16:25.427235+0000","last_became_active":"2024-09-13T14:15:22.549735+0000","last_became_peered":"2024-09-13T14:15:22.549735+0000","last_unstale":"2024-09-13T14:16:25.427235+0000","last_undegraded":"2024-09-13T14:16:25.427235+0000","last_fullsized":"2024-09-13T14:16:25.427235+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398463+0000","last_change":"2024-09-13T14:15:26.556457+0000","last_active":"2024-09-13T14:16:25.398463+0000","last_peered":"2024-09-13T14:16:25.398463+0000","last_clean":"2024-09-13T14:16:25.398463+0000","last_became_active":"2024-09-13T14:15:26.555684+0000","last_became_peered":"2024-09-13T14:15:26.555684+0000","last_unstale":"2024-09-13T14:16:25.398463+0000","last_undegraded":"2024-09-13T14:16:25.398463+0000","last_fullsized":"2024-09-13T14:16:25.398463+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402595+0000","last_change":"2024-09-13T14:15:22.549793+0000","last_active":"2024-09-13T14:16:25.402595+0000","last_peered":"2024-09-13T14:16:25.402595+0000","last_clean":"2024-09-13T14:16:25.402595+0000","last_became_active":"2024-09-13T14:15:22.549291+0000","last_became_peered":"2024-09-13T14:15:22.549291+0000","last_unstale":"2024-09-13T14:16:25.402595+0000","last_undegraded":"2024-09-13T14:16:25.402595+0000","last_fullsized":"2024-09-13T14:16:25.402595+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.399236+0000","last_change":"2024-09-13T14:15:26.554955+0000","last_active":"2024-09-13T14:16:25.399236+0000","last_peered":"2024-09-13T14:16:25.399236+0000","last_clean":"2024-09-13T14:16:25.399236+0000","last_became_active":"2024-09-13T14:15:26.553125+0000","last_became_peered":"2024-09-13T14:15:26.553125+0000","last_unstale":"2024-09-13T14:16:25.399236+0000","last_undegraded":"2024-09-13T14:16:25.399236+0000","last_fullsized":"2024-09-13T14:16:25.399236+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.402834+0000","last_change":"2024-09-13T14:15:22.549798+0000","last_active":"2024-09-13T14:16:25.402834+0000","last_peered":"2024-09-13T14:16:25.402834+0000","last_clean":"2024-09-13T14:16:25.402834+0000","last_became_active":"2024-09-13T14:15:22.549508+0000","last_became_peered":"2024-09-13T14:15:22.549508+0000","last_unstale":"2024-09-13T14:16:25.402834+0000","last_undegraded":"2024-09-13T14:16:25.402834+0000","last_fullsized":"2024-09-13T14:16:25.402834+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400073+0000","last_change":"2024-09-13T14:15:26.571783+0000","last_active":"2024-09-13T14:16:25.400073+0000","last_peered":"2024-09-13T14:16:25.400073+0000","last_clean":"2024-09-13T14:16:25.400073+0000","last_became_active":"2024-09-13T14:15:26.571523+0000","last_became_peered":"2024-09-13T14:15:26.571523+0000","last_unstale":"2024-09-13T14:16:25.400073+0000","last_undegraded":"2024-09-13T14:16:25.400073+0000","last_fullsized":"2024-09-13T14:16:25.400073+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.427718+0000","last_change":"2024-09-13T14:15:22.549324+0000","last_active":"2024-09-13T14:16:25.427718+0000","last_peered":"2024-09-13T14:16:25.427718+0000","last_clean":"2024-09-13T14:16:25.427718+0000","last_became_active":"2024-09-13T14:15:22.549022+0000","last_became_peered":"2024-09-13T14:15:22.549022+0000","last_unstale":"2024-09-13T14:16:25.427718+0000","last_undegraded":"2024-09-13T14:16:25.427718+0000","last_fullsized":"2024-09-13T14:16:25.427718+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.425792+0000","last_change":"2024-09-13T14:15:26.549648+0000","last_active":"2024-09-13T14:16:25.425792+0000","last_peered":"2024-09-13T14:16:25.425792+0000","last_clean":"2024-09-13T14:16:25.425792+0000","last_became_active":"2024-09-13T14:15:26.549328+0000","last_became_peered":"2024-09-13T14:15:26.549328+0000","last_unstale":"2024-09-13T14:16:25.425792+0000","last_undegraded":"2024-09-13T14:16:25.425792+0000","last_fullsized":"2024-09-13T14:16:25.425792+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.405098+0000","last_change":"2024-09-13T14:15:26.573588+0000","last_active":"2024-09-13T14:16:25.405098+0000","last_peered":"2024-09-13T14:16:25.405098+0000","last_clean":"2024-09-13T14:16:25.405098+0000","last_became_active":"2024-09-13T14:15:26.572830+0000","last_became_peered":"2024-09-13T14:15:26.572830+0000","last_unstale":"2024-09-13T14:16:25.405098+0000","last_undegraded":"2024-09-13T14:16:25.405098+0000","last_fullsized":"2024-09-13T14:16:25.405098+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426524+0000","last_change":"2024-09-13T14:15:22.550180+0000","last_active":"2024-09-13T14:16:25.426524+0000","last_peered":"2024-09-13T14:16:25.426524+0000","last_clean":"2024-09-13T14:16:25.426524+0000","last_became_active":"2024-09-13T14:15:22.549754+0000","last_became_peered":"2024-09-13T14:15:22.549754+0000","last_unstale":"2024-09-13T14:16:25.426524+0000","last_undegraded":"2024-09-13T14:16:25.426524+0000","last_fullsized":"2024-09-13T14:16:25.426524+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.398480+0000","last_change":"2024-09-13T14:15:46.963486+0000","last_active":"2024-09-13T14:16:25.398480+0000","last_peered":"2024-09-13T14:16:25.398480+0000","last_clean":"2024-09-13T14:16:25.398480+0000","last_became_active":"2024-09-13T14:15:46.963081+0000","last_became_peered":"2024-09-13T14:15:46.963081+0000","last_unstale":"2024-09-13T14:16:25.398480+0000","last_undegraded":"2024-09-13T14:16:25.398480+0000","last_fullsized":"2024-09-13T14:16:25.398480+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:51.607549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426984+0000","last_change":"2024-09-13T14:15:22.550265+0000","last_active":"2024-09-13T14:16:25.426984+0000","last_peered":"2024-09-13T14:16:25.426984+0000","last_clean":"2024-09-13T14:16:25.426984+0000","last_became_active":"2024-09-13T14:15:22.549179+0000","last_became_peered":"2024-09-13T14:15:22.549179+0000","last_unstale":"2024-09-13T14:16:25.426984+0000","last_undegraded":"2024-09-13T14:16:25.426984+0000","last_fullsized":"2024-09-13T14:16:25.426984+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426376+0000","last_change":"2024-09-13T14:15:26.575732+0000","last_active":"2024-09-13T14:16:25.426376+0000","last_peered":"2024-09-13T14:16:25.426376+0000","last_clean":"2024-09-13T14:16:25.426376+0000","last_became_active":"2024-09-13T14:15:26.575235+0000","last_became_peered":"2024-09-13T14:15:26.575235+0000","last_unstale":"2024-09-13T14:16:25.426376+0000","last_undegraded":"2024-09-13T14:16:25.426376+0000","last_fullsized":"2024-09-13T14:16:25.426376+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400237+0000","last_change":"2024-09-13T14:15:22.542901+0000","last_active":"2024-09-13T14:16:25.400237+0000","last_peered":"2024-09-13T14:16:25.400237+0000","last_clean":"2024-09-13T14:16:25.400237+0000","last_became_active":"2024-09-13T14:15:22.542604+0000","last_became_peered":"2024-09-13T14:15:22.542604+0000","last_unstale":"2024-09-13T14:16:25.400237+0000","last_undegraded":"2024-09-13T14:16:25.400237+0000","last_fullsized":"2024-09-13T14:16:25.400237+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.400092+0000","last_change":"2024-09-13T14:15:26.555625+0000","last_active":"2024-09-13T14:16:25.400092+0000","last_peered":"2024-09-13T14:16:25.400092+0000","last_clean":"2024-09-13T14:16:25.400092+0000","last_became_active":"2024-09-13T14:15:26.554633+0000","last_became_peered":"2024-09-13T14:15:26.554633+0000","last_unstale":"2024-09-13T14:16:25.400092+0000","last_undegraded":"2024-09-13T14:16:25.400092+0000","last_fullsized":"2024-09-13T14:16:25.400092+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":44,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:16:25.426535+0000","last_change":"2024-09-13T14:15:46.609525+0000","last_active":"2024-09-13T14:16:25.426535+0000","last_peered":"2024-09-13T14:16:25.426535+0000","last_clean":"2024-09-13T14:16:25.426535+0000","last_became_active":"2024-09-13T14:15:46.605752+0000","last_became_peered":"2024-09-13T14:15:46.605752+0000","last_unstale":"2024-09-13T14:16:25.426535+0000","last_undegraded":"2024-09-13T14:16:25.426535+0000","last_fullsized":"2024-09-13T14:16:25.426535+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:29.004573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":900361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.67400000000000004,"5min":0.749,"15min":0.749},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.56000000000000005,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.52200000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63800000000000001,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.40300000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.56100000000000005,"15min":0.56899999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.69299999999999995,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.47099999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.498},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58099999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64800000000000002,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55600000000000005}]},{"osd":4,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57799999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46600000000000003,"15min":0.47399999999999998},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66900000000000004,"5min":0.745,"15min":0.745},"last":0.45000000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.48899999999999999,"15min":0.497},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65500000000000003,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.373},{"interface":"front","average":{"1min":0.44,"5min":0.49299999999999999,"15min":0.502},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64000000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.48599999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.503,"15min":0.50900000000000001},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68100000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.432},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.502,"15min":0.51000000000000001},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.57299999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.38700000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626469,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":902387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54400000000000004,"15min":0.55200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.47899999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75900000000000001,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.57199999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53500000000000003,"15min":0.54000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78000000000000003,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.39000000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.371}]},{"osd":4,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.28199999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60299999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.45600000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.66100000000000003,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.51500000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:17:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.495,"15min":0.495},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.69599999999999995,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.44400000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822697,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":311792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51900000000000002,"15min":0.53200000000000003},"min":{"1min":0.40600000000000003,"5min":0.312,"15min":0.312},"max":{"1min":0.64600000000000002,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54300000000000004,"15min":0.54200000000000004},"min":{"1min":0.35999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70899999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.35299999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69899999999999995,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.441},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.36799999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.56000000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.39700000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67900000000000005,"5min":0.755,"15min":0.755},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73799999999999999,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52700000000000002,"15min":0.53900000000000003},"min":{"1min":0.36499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65400000000000003,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.68400000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54700000000000004}]},{"osd":4,"last update":"Fri Sep 13 14:17:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.40799999999999997,"15min":0.39400000000000002},"min":{"1min":0.307,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66400000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.35999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.53800000000000003,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.47399999999999998,"15min":0.47899999999999998},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.46400000000000002,"15min":0.46600000000000003},"min":{"1min":0.28899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.498}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.51700000000000002,"15min":0.52900000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61299999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55300000000000005,"15min":0.56299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71299999999999997,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57899999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56000000000000005,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.63700000000000001,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.44},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68700000000000006,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.50600000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65700000000000003,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.495},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.56200000000000006,"15min":0.57599999999999996},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66800000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.60699999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.57599999999999996,"15min":0.58199999999999996},"min":{"1min":0.41999999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.628,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.56200000000000006,"15min":0.57099999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.61599999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.68899999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:16:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.58499999999999996,"15min":0.59499999999999997},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.66100000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63400000000000001,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.59299999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:17:15 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54100000000000004,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.39000000000000001,"15min":0.375},"min":{"1min":0.26300000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.371}]},{"osd":6,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.41299999999999998,"15min":0.41499999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.45400000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65000000000000002,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.41599999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182452,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":312417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":0.623,"5min":0.628,"15min":0.628},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.222,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34100000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.58399999999999996,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.39200000000000002,"15min":0.38800000000000001},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.27300000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.40500000000000003,"15min":0.40400000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.32500000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:16:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59499999999999997,"15min":0.57699999999999996},"min":{"1min":0.442,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86499999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.59999999999999998,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.86599999999999999,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.496}]},{"osd":5,"last update":"Fri Sep 13 14:17:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.54900000000000004},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.874,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57999999999999996,"15min":0.56999999999999995},"min":{"1min":0.435,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.48199999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:16:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.46400000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95299999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.44500000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:16:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.433,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.88,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.505},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.63700000000000001,"15min":0.64300000000000002},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.85899999999999999,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.51400000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411384,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":692224,"data_stored":310541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.67900000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.40899999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.57199999999999995,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.40500000000000003},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.439}]},{"osd":3,"last update":"Fri Sep 13 14:16:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45700000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73099999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64800000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.46700000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:16:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78800000000000003,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.57999999999999996,"15min":0.58099999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71499999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.54000000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:17:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.63300000000000001,"15min":0.64200000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80700000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57499999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:16:33 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60799999999999998,"15min":0.60699999999999998},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.626},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63900000000000001,"15min":0.64400000000000002},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.56000000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:16:57 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.65900000000000003,"15min":0.66500000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84999999999999998,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83999999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.54700000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":692224,"data_stored":309520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:16:37 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.38700000000000001,"15min":0.34699999999999998},"min":{"1min":0.309,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.54900000000000004,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.434,"15min":0.379},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.67600000000000005,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.38200000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:17:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.39900000000000002,"15min":0.375},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.64400000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41799999999999998,"15min":0.39400000000000002},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.33800000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:16:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":0.63700000000000001,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61099999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.50700000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:16:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58699999999999997,"15min":0.57399999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.56499999999999995,"15min":0.55400000000000005},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.56100000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:17:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.63700000000000001,"15min":0.65200000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.437},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.53200000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:16:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.45100000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.65700000000000003,"15min":0.66000000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.629,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.85399999999999998,"5min":0.87,"15min":0.87},"last":0.58899999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738434,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":901055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.27800000000000002,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":0.54000000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.37,"15min":0.34899999999999998},"min":{"1min":0.24299999999999999,"5min":0.184,"15min":0.184},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.38400000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:17:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39600000000000002,"15min":0.38400000000000001},"min":{"1min":0.29899999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.46800000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.433},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48699999999999999,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.441}]},{"osd":3,"last update":"Fri Sep 13 14:16:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.47299999999999998,"15min":0.47199999999999998},"min":{"1min":0.34699999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65900000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.57999999999999996,"5min":0.625,"15min":0.625},"last":0.40000000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:16:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.55600000000000005,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79100000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.88100000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:17:18 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56499999999999995,"15min":0.54900000000000004},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81200000000000006,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.375,"15min":0.375},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.75700000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.47499999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.748},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.76500000000000001,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.79300000000000004}]},{"osd":7,"last update":"Fri Sep 13 14:17:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58999999999999997,"15min":0.59299999999999997},"min":{"1min":0.438,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.47899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.73899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:17:27.734 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:17:27.734 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:17:27.734 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:17:27.734 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:17:27.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:27 smithi026 ceph-mon[39182]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:27.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:27 smithi026 ceph-mon[27770]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:27 smithi145 ceph-mon[31057]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:17:28.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:28 smithi026 ceph-mon[27770]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:28.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:28 smithi026 ceph-mon[39182]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:28 smithi145 ceph-mon[31057]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.199 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:30.326 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:30 smithi026 systemd[1]: Stopping Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:30.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:30 smithi026 ceph-mon[27770]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:30 smithi026 ceph-mon[39182]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:30 smithi145 ceph-mon[31057]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:31.665 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[73282]: ts=2024-09-13T14:17:31.499Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:17:31.665 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:31 smithi026 podman[78975]: 2024-09-13 14:17:31.520817947 +0000 UTC m=+0.560664360 container died e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:32.161 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:31 smithi026 podman[78975]: 2024-09-13 14:17:31.898036476 +0000 UTC m=+0.937882889 container cleanup e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:32.161 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:31 smithi026 bash[78975]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:17:32.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:32 smithi026 ceph-mon[39182]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:32.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:32 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:32.705 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:32 smithi026 podman[79012]: 2024-09-13 14:17:32.435721617 +0000 UTC m=+0.913332244 container remove e103422c5212927b6991944b0219fa84cbeb9b4b9bbf51f7d87607ab317c4f44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:32.705 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:32 smithi026 podman[79012]: 2024-09-13 14:17:32.536727105 +0000 UTC m=+1.014337730 volume remove 2832aae069ecbbcf85e25e2ed0897728a3e524a8d5301af2a370ee52d87c0317 2024-09-13T14:17:32.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:32 smithi026 ceph-mon[27770]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:32.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:32 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:32 smithi145 ceph-mon[31057]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:32 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:32.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:17:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:17:33.017 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:33.017 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:17:33.017 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:32 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:17:33.018 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:32 smithi026 systemd[1]: Stopped Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:33.018 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:32 smithi026 systemd[1]: Starting Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:33.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/411059196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:17:33.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/411059196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:17:33.602 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 podman[79156]: 2024-09-13 14:17:33.31489548 +0000 UTC m=+0.286856736 volume create 4e9b57ae05541b8d878c090dcc2766befb29fa949f97d71f6d331ca37f5719ac 2024-09-13T14:17:33.603 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 podman[79156]: 2024-09-13 14:17:33.225006627 +0000 UTC m=+0.196967886 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T14:17:33.603 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 podman[79156]: 2024-09-13 14:17:33.415791437 +0000 UTC m=+0.387752736 container create 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:33 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/411059196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 podman[79156]: 2024-09-13 14:17:33.710242938 +0000 UTC m=+0.682204194 container init 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 podman[79156]: 2024-09-13 14:17:33.715021156 +0000 UTC m=+0.686982415 container start 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.728Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.728Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.729Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.730Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.753Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.753Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.756Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:33.756Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:17:33.964 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 bash[79156]: 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 2024-09-13T14:17:33.965 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:33 smithi026 systemd[1]: Started Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:34.435 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:17:34.435 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:17:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[27770]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[39182]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:34 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:34.700 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:34.827 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:34 smithi145 ceph-mon[31057]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.827 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:34 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:34.827 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:34 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:35.674 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:35 smithi145 ceph-mon[31057]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:35.675 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:35 smithi145 ceph-mon[31057]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:17:35.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:35 smithi026 ceph-mon[39182]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:35.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:35 smithi026 ceph-mon[39182]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:17:35.830 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:35.730Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000345206s 2024-09-13T14:17:35.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:35 smithi026 ceph-mon[27770]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:35.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:35 smithi026 ceph-mon[27770]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:17:36.563 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:36.564 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14421,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.26:6835/248064250","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":248064250},{"type":"v1","addr":"172.21.15.26:6835","nonce":248064250}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:15:41.581255+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24331},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24331":{"gid":24331,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.145:6833/2983197144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2983197144},{"type":"v1","addr":"172.21.15.145:6833","nonce":2983197144}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-13T14:17:36.564 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 8 2024-09-13T14:17:36.673 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:36 smithi145 ceph-mon[31057]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:36 smithi026 ceph-mon[39182]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:36 smithi026 ceph-mon[27770]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:36 smithi145 systemd[1]: Stopping Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:37.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.070Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:17:37.353 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.071Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:17:37.354 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.071Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:17:37.354 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[60949]: ts=2024-09-13T14:17:37.071Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:17:37.354 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 podman[64321]: 2024-09-13 14:17:37.080801597 +0000 UTC m=+0.122627718 container died 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:37.533 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:37 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/250417240' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:37.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 podman[64321]: 2024-09-13 14:17:37.460077406 +0000 UTC m=+0.501903528 container cleanup 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:37.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 bash[64321]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:17:37.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:37 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/250417240' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:37.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:37 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/250417240' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:38.087 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 podman[64338]: 2024-09-13 14:17:37.71973701 +0000 UTC m=+0.641230587 container remove 108aa9bd14c5f8f00b8929d0dc004ed57159932e9d99048b02fe771f3069b9f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:38.087 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:17:38.088 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 systemd[1]: Stopped Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:38.088 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.091s CPU time. 2024-09-13T14:17:38.088 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:37 smithi145 systemd[1]: Starting Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 podman[64438]: 2024-09-13 14:17:37.990837404 +0000 UTC m=+0.017147319 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 podman[64438]: 2024-09-13 14:17:38.095508393 +0000 UTC m=+0.121818346 container create 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 podman[64438]: 2024-09-13 14:17:38.297703944 +0000 UTC m=+0.324013855 container init 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 podman[64438]: 2024-09-13 14:17:38.302244097 +0000 UTC m=+0.328554009 container start 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.324Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.324Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.324Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi145 (none))" 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.325Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.325Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:17:38.347 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.326Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.326Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.328Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.328Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.329Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.329Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.505µs 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.329Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.330Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.330Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.330Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.803µs wal_replay_duration=412.998µs wbl_replay_duration=94ns total_replay_duration=457.259µs 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.331Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.331Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:17:38.348 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.331Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:17:38.349 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.346Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.748984ms db_storage=1.083µs remote_storage=1.046µs web_handler=423ns query_engine=575ns scrape=559.52µs scrape_sd=24.145µs notify=35.062µs notify_sd=5.289µs rules=13.881889ms tracing=6.277µs 2024-09-13T14:17:38.349 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.346Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:17:38.349 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:17:38.346Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:17:38.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 bash[64438]: 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda 2024-09-13T14:17:38.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:17:38 smithi145 systemd[1]: Started Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:38.745 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Bus STOPPING 2024-09-13T14:17:38.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:38.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:38.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:38.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Bus STOPPED 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Bus STARTING 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Serving on http://:::9283 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Bus STARTED 2024-09-13T14:17:39.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:38] ENGINE Bus STOPPING 2024-09-13T14:17:39.489 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:39.489 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14421,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.26:6835/248064250","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":248064250},{"type":"v1","addr":"172.21.15.26:6835","nonce":248064250}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:15:41.581255+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24331},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24331":{"gid":24331,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.145:6833/2983197144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2983197144},{"type":"v1","addr":"172.21.15.145:6833","nonce":2983197144}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-13T14:17:39.490 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 8 2024-09-13T14:17:39.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/191519489' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:39.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/191519489' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STOPPED 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STARTING 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Serving on http://:::9283 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STARTED 2024-09-13T14:17:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STOPPING 2024-09-13T14:17:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi145.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi145.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:17:39.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:39 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/191519489' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:40.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:17:40.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STOPPED 2024-09-13T14:17:40.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:39 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:39] ENGINE Bus STARTING 2024-09-13T14:17:40.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:40] ENGINE Serving on http://:::9283 2024-09-13T14:17:40.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:40] ENGINE Bus STARTED 2024-09-13T14:17:40.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:40 smithi026 ceph-mon[39182]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:40.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:40 smithi026 ceph-mon[27770]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:40.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:17:40.907 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:40 smithi145 ceph-mon[31057]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:41.063 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[39182]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[27770]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:42.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:42 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:42 smithi145 ceph-mon[31057]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:42 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:42 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:43.405 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":67,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:16:33.688272+0000","last_up_change":"2024-09-13T14:14:55.454576+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-13T14:16:19.892063+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6803","nonce":632585262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6805","nonce":632585262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6809","nonce":632585262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":632585262},{"type":"v1","addr":"172.21.15.26:6807","nonce":632585262}]},"public_addr":"172.21.15.26:6803/632585262","cluster_addr":"172.21.15.26:6805/632585262","heartbeat_back_addr":"172.21.15.26:6809/632585262","heartbeat_front_addr":"172.21.15.26:6807/632585262","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6811","nonce":3428658120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6813","nonce":3428658120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6817","nonce":3428658120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3428658120},{"type":"v1","addr":"172.21.15.26:6815","nonce":3428658120}]},"public_addr":"172.21.15.26:6811/3428658120","cluster_addr":"172.21.15.26:6813/3428658120","heartbeat_back_addr":"172.21.15.26:6817/3428658120","heartbeat_front_addr":"172.21.15.26:6815/3428658120","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6819","nonce":3032986048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6821","nonce":3032986048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6825","nonce":3032986048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3032986048},{"type":"v1","addr":"172.21.15.26:6823","nonce":3032986048}]},"public_addr":"172.21.15.26:6819/3032986048","cluster_addr":"172.21.15.26:6821/3032986048","heartbeat_back_addr":"172.21.15.26:6825/3032986048","heartbeat_front_addr":"172.21.15.26:6823/3032986048","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6827","nonce":3933565327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6829","nonce":3933565327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6833","nonce":3933565327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3933565327},{"type":"v1","addr":"172.21.15.26:6831","nonce":3933565327}]},"public_addr":"172.21.15.26:6827/3933565327","cluster_addr":"172.21.15.26:6829/3933565327","heartbeat_back_addr":"172.21.15.26:6833/3933565327","heartbeat_front_addr":"172.21.15.26:6831/3933565327","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6801","nonce":2159398871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6803","nonce":2159398871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6807","nonce":2159398871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2159398871},{"type":"v1","addr":"172.21.15.145:6805","nonce":2159398871}]},"public_addr":"172.21.15.145:6801/2159398871","cluster_addr":"172.21.15.145:6803/2159398871","heartbeat_back_addr":"172.21.15.145:6807/2159398871","heartbeat_front_addr":"172.21.15.145:6805/2159398871","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6809","nonce":2344369839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6811","nonce":2344369839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6815","nonce":2344369839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":2344369839},{"type":"v1","addr":"172.21.15.145:6813","nonce":2344369839}]},"public_addr":"172.21.15.145:6809/2344369839","cluster_addr":"172.21.15.145:6811/2344369839","heartbeat_back_addr":"172.21.15.145:6815/2344369839","heartbeat_front_addr":"172.21.15.145:6813/2344369839","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6817","nonce":913657777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6819","nonce":913657777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6823","nonce":913657777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":913657777},{"type":"v1","addr":"172.21.15.145:6821","nonce":913657777}]},"public_addr":"172.21.15.145:6817/913657777","cluster_addr":"172.21.15.145:6819/913657777","heartbeat_back_addr":"172.21.15.145:6823/913657777","heartbeat_front_addr":"172.21.15.145:6821/913657777","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6825","nonce":2626541206}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6827","nonce":2626541206}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6831","nonce":2626541206}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2626541206},{"type":"v1","addr":"172.21.15.145:6829","nonce":2626541206}]},"public_addr":"172.21.15.145:6825/2626541206","cluster_addr":"172.21.15.145:6827/2626541206","heartbeat_back_addr":"172.21.15.145:6831/2626541206","heartbeat_front_addr":"172.21.15.145:6829/2626541206","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6800/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/1842776456":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/2247322222":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/173360804":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3442380687":"2024-09-14T14:16:17.348500+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:17:43.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:43 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2448121010' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:17:43.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:43 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2448121010' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:17:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:43 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2448121010' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:17:44.026 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:17:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:17:43.733Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002912069s 2024-09-13T14:17:44.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:17:44.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:44 smithi026 ceph-mon[39182]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:44.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:44 smithi026 ceph-mon[27770]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:44.876 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:44 smithi145 ceph-mon[31057]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:45.283 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:45] "GET /metrics HTTP/1.1" 200 236503 "" "Prometheus/2.43.0" 2024-09-13T14:17:46.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[39182]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:46.164 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:45 smithi026 ceph-mon[27770]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:45 smithi145 ceph-mon[31057]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:46.885 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:17:46.885 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14421,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.26:6835/248064250","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":248064250},{"type":"v1","addr":"172.21.15.26:6835","nonce":248064250}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:15:41.581255+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24331},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24331":{"gid":24331,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.145:6833/2983197144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2983197144},{"type":"v1","addr":"172.21.15.145:6833","nonce":2983197144}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-13T14:17:46.885 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 8 2024-09-13T14:17:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:46 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/933677055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:46 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/933677055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:46 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/933677055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.666 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 8, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-13T14:15:28.665431+0000', 'modified': '2024-09-13T14:15:41.581255+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24331}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24331': {'gid': 24331, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.145:6833/2983197144', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.145:6832', 'nonce': 2983197144}, {'type': 'v1', 'addr': '172.21.15.145:6833', 'nonce': 2983197144}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-13T14:17:47.666 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-13T14:17:47.667 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph tell mds.b status 2024-09-13T14:17:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:47 smithi145 ceph-mon[31057]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:47 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:48.258 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:48.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:47 smithi026 ceph-mon[27770]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:48.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:47 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:48.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:47 smithi026 ceph-mon[39182]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:48.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:47 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:49.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:49 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:49 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:49 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:49 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:49 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:49 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:49.955 INFO:teuthology.orchestra.run.smithi026.stderr:2024-09-13T14:17:49.953+0000 7fe443fff640 0 client.14757 ms_handle_reset on v2:172.21.15.145:6832/2983197144 2024-09-13T14:17:49.973 INFO:teuthology.orchestra.run.smithi026.stderr:2024-09-13T14:17:49.972+0000 7fe443fff640 0 client.24659 ms_handle_reset on v2:172.21.15.145:6832/2983197144 2024-09-13T14:17:49.973 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:17:49.973 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_fsid": "ad455616-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "whoami": 0, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "id": 24331, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "want_state": "up:active", 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "state": "up:active", 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "fs_name": "cephfs", 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "rank_uptime": 140.41376610099999, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "mdsmap_epoch": 8, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "osdmap_epoch": 67, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "osdmap_epoch_barrier": 57, 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout: "uptime": 157.76224299500001 2024-09-13T14:17:49.974 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:17:50.733 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T14:17:50.733 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:17:50.743 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-13T14:17:50.743 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:17:50.766 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-13T14:17:50.766 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T14:17:50.840 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-09-13T14:17:50.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-13T14:17:50.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:50 smithi026 ceph-mon[27770]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:50.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:50 smithi026 ceph-mon[27770]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-13T14:17:50.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:50 smithi026 ceph-mon[39182]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:50.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:50 smithi026 ceph-mon[39182]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-13T14:17:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:50 smithi145 ceph-mon[31057]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:50 smithi145 ceph-mon[31057]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-13T14:17:51.559 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:52 smithi026 ceph-mon[27770]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:52.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:52 smithi026 ceph-mon[39182]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:52.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:52 smithi145 ceph-mon[31057]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:52.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:17:52 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:53.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:53.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:53 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:53 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:53 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:53 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:53 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:53 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:54.504 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE Bus STOPPING 2024-09-13T14:17:54.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:54.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' 2024-09-13T14:17:54.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:54.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:54.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:54.798 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:17:54.799 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE Bus STOPPED 2024-09-13T14:17:54.799 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE Bus STARTING 2024-09-13T14:17:54.799 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE Serving on http://:::9283 2024-09-13T14:17:54.799 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:17:54] ENGINE Bus STARTED 2024-09-13T14:17:54.822 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:17:54.831 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-13T14:17:54.831 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:17:54.868 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-13T14:17:54.868 INFO:teuthology.run_tasks:Running task workunit... 2024-09-13T14:17:54.916 INFO:tasks.workunit:Pulling workunits from ref d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:17:54.916 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:17:54.916 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:17:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:17:54.934 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:17:54.934 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:54 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:17:54.990 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:17:54.991 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:17:55.047 INFO:tasks.workunit:timeout=3h 2024-09-13T14:17:55.048 INFO:tasks.workunit:cleanup=True 2024-09-13T14:17:55.048 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:17:55.106 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:17:55.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:17:55 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:17:55] "GET /metrics HTTP/1.1" 200 236497 "" "Prometheus/2.43.0" 2024-09-13T14:17:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:56 smithi145 ceph-mon[31057]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:57.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:56 smithi026 ceph-mon[27770]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:57.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:56 smithi026 ceph-mon[39182]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:17:58 smithi145 ceph-mon[31057]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:59.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:17:58 smithi026 ceph-mon[27770]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:17:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:17:58 smithi026 ceph-mon[39182]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:18:01.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:00 smithi026 ceph-mon[39182]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:18:01.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:00 smithi026 ceph-mon[27770]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:18:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:00 smithi145 ceph-mon[31057]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:18:03.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:02 smithi026 ceph-mon[39182]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:03.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:02 smithi026 ceph-mon[27770]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:02 smithi145 ceph-mon[31057]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:03.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:05.007 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:04 smithi145 ceph-mon[31057]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:05.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:04 smithi026 ceph-mon[27770]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:05.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:04 smithi026 ceph-mon[39182]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:05.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:05] "GET /metrics HTTP/1.1" 200 236510 "" "Prometheus/2.43.0" 2024-09-13T14:18:07.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:06 smithi026 ceph-mon[27770]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:07.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:06 smithi026 ceph-mon[39182]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:06 smithi145 ceph-mon[31057]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:09.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:08 smithi026 ceph-mon[27770]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:09.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:08 smithi026 ceph-mon[39182]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:08 smithi145 ceph-mon[31057]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:10 smithi026 ceph-mon[27770]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:11.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:10 smithi026 ceph-mon[39182]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:10 smithi145 ceph-mon[31057]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:13.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:12 smithi026 ceph-mon[39182]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:13.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:12 smithi026 ceph-mon[27770]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:12 smithi145 ceph-mon[31057]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:13.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:14.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:14 smithi026 ceph-mon[39182]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:14.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:14 smithi026 ceph-mon[27770]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:14 smithi145 ceph-mon[31057]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:15.237 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:15] "GET /metrics HTTP/1.1" 200 236510 "" "Prometheus/2.43.0" 2024-09-13T14:18:17.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:16 smithi026 ceph-mon[39182]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:17.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:16 smithi026 ceph-mon[27770]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:16 smithi145 ceph-mon[31057]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:18.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:18 smithi026 ceph-mon[39182]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:18 smithi026 ceph-mon[27770]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:18 smithi145 ceph-mon[31057]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:20 smithi026 ceph-mon[39182]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:20 smithi026 ceph-mon[27770]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:20 smithi145 ceph-mon[31057]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:23.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:22 smithi026 ceph-mon[39182]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:22 smithi026 ceph-mon[27770]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:22 smithi145 ceph-mon[31057]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:23.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:25.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:24 smithi026 ceph-mon[39182]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:25.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:24 smithi026 ceph-mon[27770]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:24 smithi145 ceph-mon[31057]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:25.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:25 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:25] "GET /metrics HTTP/1.1" 200 236502 "" "Prometheus/2.43.0" 2024-09-13T14:18:27.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:26 smithi026 ceph-mon[39182]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:26 smithi026 ceph-mon[27770]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:26 smithi145 ceph-mon[31057]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:29.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:28 smithi026 ceph-mon[39182]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:28 smithi026 ceph-mon[27770]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:28 smithi145 ceph-mon[31057]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:31.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:30 smithi026 ceph-mon[27770]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:30 smithi026 ceph-mon[39182]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:30 smithi145 ceph-mon[31057]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:32 smithi026 ceph-mon[39182]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:32 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:32 smithi026 ceph-mon[27770]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:32 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:32 smithi145 ceph-mon[31057]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:32 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:33.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:35.007 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:34 smithi145 ceph-mon[31057]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:35.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:34 smithi026 ceph-mon[39182]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:35.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:34 smithi026 ceph-mon[27770]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:35.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:35] "GET /metrics HTTP/1.1" 200 236499 "" "Prometheus/2.43.0" 2024-09-13T14:18:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:36 smithi026 ceph-mon[39182]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:37.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:36 smithi026 ceph-mon[27770]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:36 smithi145 ceph-mon[31057]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:38 smithi026 ceph-mon[39182]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:38 smithi026 ceph-mon[27770]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:38 smithi145 ceph-mon[31057]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:41.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:40 smithi026 ceph-mon[39182]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:40 smithi026 ceph-mon[27770]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:40 smithi145 ceph-mon[31057]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:43.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:42 smithi026 ceph-mon[39182]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:42 smithi026 ceph-mon[27770]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:42 smithi145 ceph-mon[31057]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:45.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:44 smithi026 ceph-mon[39182]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:45.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:44 smithi026 ceph-mon[27770]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:45.024 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:45] "GET /metrics HTTP/1.1" 200 236499 "" "Prometheus/2.43.0" 2024-09-13T14:18:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:44 smithi145 ceph-mon[31057]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:46 smithi026 ceph-mon[39182]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:46 smithi026 ceph-mon[27770]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:46 smithi145 ceph-mon[31057]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:48.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:47 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:47 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:47 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:49.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:48 smithi026 ceph-mon[39182]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:48 smithi026 ceph-mon[27770]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:48 smithi145 ceph-mon[31057]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:50 smithi026 ceph-mon[39182]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:50 smithi026 ceph-mon[27770]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:50 smithi145 ceph-mon[31057]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:52 smithi026 ceph-mon[39182]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:52 smithi026 ceph-mon[27770]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:52 smithi145 ceph-mon[31057]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:18:52 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:18:55.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[27770]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:55.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[39182]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:54 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:55 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:18:55] "GET /metrics HTTP/1.1" 200 236470 "" "Prometheus/2.43.0" 2024-09-13T14:18:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:54 smithi145 ceph-mon[31057]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:54 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:54 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:56.044 INFO:tasks.workunit.client.0.smithi026.stderr:Updating files: 91% (11644/12697) Updating files: 92% (11682/12697) Updating files: 93% (11809/12697) Updating files: 94% (11936/12697) Updating files: 95% (12063/12697) Updating files: 96% (12190/12697) Updating files: 97% (12317/12697) Updating files: 98% (12444/12697) Updating files: 99% (12571/12697) Updating files: 100% (12697/12697) Updating files: 100% (12697/12697), done. 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:Note: switching to 'd58f73069a8157a9ace12a0247ee3e8497707ea9'. 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:state without impacting any branches by switching back to a branch. 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-13T14:18:56.369 INFO:tasks.workunit.client.0.smithi026.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: git switch -c 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr:Or undo this operation with: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: git switch - 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:18:56.370 INFO:tasks.workunit.client.0.smithi026.stderr:HEAD is now at d58f73069a8 tools/ceph-bluestore-tool: Added zap-device documentation 2024-09-13T14:18:56.374 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:18:56.437 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:18:56.439 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:18:56.439 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:18:57.138 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:18:57.150 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:56 smithi026 ceph-mon[39182]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:56 smithi026 ceph-mon[27770]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:56 smithi145 ceph-mon[31057]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.207 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:18:57.248 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:18:57.249 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:18:57.250 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:18:57.289 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:18:57.293 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:18:57.293 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:18:57.351 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-13T14:18:57.352 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-13T14:18:57.352 DEBUG:teuthology.orchestra.run.smithi026:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d58f73069a8157a9ace12a0247ee3e8497707ea9 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-13T14:18:57.412 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-13T14:18:57.793 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-13T14:18:57.997 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:57] ENGINE Bus STOPPING 2024-09-13T14:18:58.166 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STOPPED 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STARTING 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Serving on http://:::9283 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STARTED 2024-09-13T14:18:58.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STOPPING 2024-09-13T14:18:58.549 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry preview 2024-09-13T14:18:58.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2801268151' entity='client.admin' 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/143381354' entity='client.admin' 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STOPPED 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STARTING 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Serving on http://:::9283 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STARTED 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:58] ENGINE Bus STOPPING 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2801268151' entity='client.admin' 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/143381354' entity='client.admin' 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:58 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2801268151' entity='client.admin' 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' 2024-09-13T14:18:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/143381354' entity='client.admin' 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:58.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:58 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer": { 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "active": true, 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "mode": "" 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "channels": [ 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:18:59.016 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "ident" 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "channels_available": [ 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:18:59.017 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_available": [ 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:18:59.018 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_opted_in": [ 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:18:59.019 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "config": { 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "active_changed": [ 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:18:59.020 INFO:tasks.workunit.client.0.smithi026.stdout: "client_check_pool_perm", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_gid", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_uid", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "client_quota", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "daemonize", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "keyring", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_block_size", 2024-09-13T14:18:59.021 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_cache_size", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_compression", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_log", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_write_buffer_size", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_journald", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_stderr", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:18:59.022 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_host", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:18:59.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "no_config_file", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:18:59.024 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_lim", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_res", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:18:59.025 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addr", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addrv", 2024-09-13T14:18:59.026 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_default_features", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_qos_exclude_ops", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "setgroup", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "setuser" 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_changed": [ 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:18:59.027 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_join_fs", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/allow_ptrace", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/container_init", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/migration_current", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/use_repo_digest", 2024-09-13T14:18:59.028 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ssl_server_port", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/orchestrator/orchestrator", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_ident", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/description", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/organization", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:18:59.029 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:18:59.030 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:18:59.031 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_cache_enabled", 2024-09-13T14:18:59.032 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_ops_log", 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_usage_log" 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "contact": null, 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "crashes": [], 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "created": "2024-09-13T14:09:50.810274Z", 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "crush": { 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_algs": { 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "straw2": 3 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_sizes": { 2024-09-13T14:18:59.033 INFO:tasks.workunit.client.0.smithi026.stdout: "2": 1, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "4": 2 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_types": { 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "1": 2, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "11": 1 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "compat_weight_set": false, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "device_classes": [ 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: 8 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_buckets": 6, 2024-09-13T14:18:59.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_devices": 8, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_rules": 1, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_types": 12, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_weight_sets": 0, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "tunables": { 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "allowed_bucket_algs": 54, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_fallback_tries": 0, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_tries": 0, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_total_tries": 50, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_descend_once": 1, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_stable": 1, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_vary_r": 1, 2024-09-13T14:18:59.035 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v2_rules": 0, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v3_rules": 0, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v4_buckets": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v5_rules": 0, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "legacy_tunables": 0, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "minimum_required_version": "jewel", 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "optimal_tunables": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "profile": "jewel", 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables2": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables3": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables5": 1, 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: "straw_calc_version": 1 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "description": "upgrade test cluster", 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "fs": { 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "feature_flags": { 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "enable_multiple": true, 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_enabled_multiple": true 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "filesystems": [ 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "approx_ctime": "2024-09", 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer_enabled": false, 2024-09-13T14:18:59.037 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_caps": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_dns": 10, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_inos": 13, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_subtrees": 2, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_allowed_features": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "explicitly_allowed_features": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "files": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "max_mds": 1, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "num_data_pools": 1, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "num_in": 1, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "num_mds": 1, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": 0, 2024-09-13T14:18:59.038 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_replay": 0, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_up": 1, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "snaps": 0, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "standby_count_wanted": 0 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_mds": 1, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "total_num_mds": 2 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "hosts": { 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num": 2, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mds": 2, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mgr": 2, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mon": 2, 2024-09-13T14:18:59.039 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_osd": 2 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "leaderboard": false, 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "license": "sharing-1-0", 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "metadata": { 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 2 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-13T14:18:59.040 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 2 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 2 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 2 2024-09-13T14:18:59.041 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 3 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 3 2024-09-13T14:18:59.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 3 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 3 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 8 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 8 2024-09-13T14:18:59.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 8 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 8 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_objectstore": { 2024-09-13T14:18:59.045 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": 8 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "rotational": { 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "0": 8 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "features": { 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "optional": [], 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "persistent": [ 2024-09-13T14:18:59.046 INFO:tasks.workunit.client.0.smithi026.stdout: "kraken", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "luminous", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "mimic", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap-prune", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "nautilus", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "octopus", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "pacific", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "elector-pinging", 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "quincy" 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv4_addr_mons": 6, 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv6_addr_mons": 0, 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "min_mon_release": 17, 2024-09-13T14:18:59.047 INFO:tasks.workunit.client.0.smithi026.stdout: "v1_addr_mons": 3, 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "v2_addr_mons": 3 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "organization": "ceph-qa", 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_network": false, 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "require_min_compat_client": "luminous", 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "require_osd_release": "quincy" 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": [ 2024-09-13T14:18:59.048 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 1, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 1, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 1, 2024-09-13T14:18:59.049 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 1781760, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 1781760, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-13T14:18:59.050 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 126, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 236544, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 590368, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 590368, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 1771104, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 113, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 1667072 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:18:59.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 32, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 32, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 2, 2024-09-13T14:18:59.052 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 98304, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 98304, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 22, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-13T14:18:59.053 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 3410, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 3410, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 10230, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 51, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 19456 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:18:59.054 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 64, 2024-09-13T14:18:59.055 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 64, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 3, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 0, 2024-09-13T14:18:59.056 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 0.0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 0, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 0 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.057 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:18:59.058 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 8, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 8, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 4, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 12288, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 12288, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:18:59.059 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.689964257423071e-08, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 19, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 19, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 57, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 2, 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 2048 2024-09-13T14:18:59.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": { 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "mirroring_by_pool": [ 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: false 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_images_by_pool": [ 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pools": 1 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "report_id": "bb35b373-4dfb-4d9f-9cf9-b49d389b3568", 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "report_timestamp": "2024-09-13T14:18:59.008701", 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "report_version": 1, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "rook": { 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster": { 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "external-mode": null, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "allow-multiple-per-node": null, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "count": null, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "max-id": null, 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "pvc": { 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:18:59.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "stretch": { 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "network": { 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "provider": null 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "storage": { 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "device-set": { 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "non-portable": null, 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "portable": null, 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:18:59.063 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "csi": { 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes": { 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "node": { 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes-total": null, 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "with-ceph-daemons": null, 2024-09-13T14:18:59.064 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-cephfs-plugin": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-nfs-plugin": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-rbd-plugin": null 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "storage-class": { 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "nfs": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": null, 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:18:59.065 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "services": {}, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 105, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": 4, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_by_class": { 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "ssd": { 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:18:59.066 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864, 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_bytes": 2443300864, 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_ratio": 0.003181545762345195 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:18:59.067 INFO:tasks.workunit.client.0.smithi026.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-13T14:18:59.068 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry preview-device 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "1e627828-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPED1D480GA_1e632c28-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141258": { 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:18:59.509 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "E2010325", 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e627828-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236777 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-13T14:18:59.510 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.511 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:18:59.512 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:18:59.513 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 8322763 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:18:59.514 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:18:59.515 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.516 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.517 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 15 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "0.000000" 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1010736 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:18:59.518 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 0, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 0, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.519 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 0, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 3253, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 114944730, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 545440628, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 1693655669, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 13000783575, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 0, 2024-09-13T14:18:59.520 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 20275, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 25721, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 31, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 6932 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 20275, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 25721 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.521 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.522 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: E2010325", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:18:59.523 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-13T14:18:59.524 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 31 Celsius", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 0%", 2024-09-13T14:18:59.525 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 0%", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 114,944,730 [58.8 TB]", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 545,440,628 [279 TB]", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 1,693,655,669", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 13,000,783,575", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 3,253", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 20,275", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 25,721", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 6,932", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:18:59.526 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 31 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.527 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "1e646412-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPEDMD400G4_1e64e450-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141301": { 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:18:59.528 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "8DV101H0", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e646412-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:18:59.529 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236781 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:18:59.530 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:18:59.531 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 27995823 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.533 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 49507808 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.534 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:18:59.535 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.536 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "63.999023" 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:18:59.537 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 89, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 3385, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 3399, 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 3370 2024-09-13T14:18:59.538 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 10, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 26, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 364889334, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 1834734314, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 6215216103, 2024-09-13T14:18:59.539 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 33084752516, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 11, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 56859, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 55867, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 25, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 1468 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.540 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 56859, 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 55867 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:18:59.541 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:18:59.542 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: 8DV101H0", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-13T14:18:59.543 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: " 1 - 512 8 2", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: " 2 - 512 16 2", 2024-09-13T14:18:59.544 INFO:tasks.workunit.client.0.smithi026.stdout: " 3 - 4096 0 0", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: " 4 - 4096 8 0", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: " 5 - 4096 64 0", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: " 6 - 4096 128 0", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 25 Celsius", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:18:59.545 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 10%", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 11%", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 364,889,334 [186 TB]", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 1,834,734,314 [939 TB]", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 6,215,216,103", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 33,084,752,516", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 26", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 56,859", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 55,867", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 1,468", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:18:59.546 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:18:59.547 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 25 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.548 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:18:59.549 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry preview-all 2024-09-13T14:18:59.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:18:59.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:59] ENGINE Bus STOPPED 2024-09-13T14:18:59.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:59] ENGINE Bus STARTING 2024-09-13T14:18:59.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:59] ENGINE Serving on http://:::9283 2024-09-13T14:18:59.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:18:59] ENGINE Bus STARTED 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:18:59 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout: "device_report": { 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout: "1e627828-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPED1D480GA_1e632c28-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141258": { 2024-09-13T14:18:59.995 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "E2010325", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e627828-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:18:59.996 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236777 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:18:59.997 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.998 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:18:59.999 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 8322763 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:00.000 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.005 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 15 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "0.000000" 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1010736 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 0, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 0, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 0, 2024-09-13T14:19:00.007 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 3253, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 114944730, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 545440628, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 1693655669, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 13000783575, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 0, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 20275, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 25721, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 31, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 6932 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:00.008 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 20275, 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 25721 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.009 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.010 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: E2010325", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:00.011 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.012 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 31 Celsius", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 0%", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 0%", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 114,944,730 [58.8 TB]", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 545,440,628 [279 TB]", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 1,693,655,669", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 13,000,783,575", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 3,253", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 20,275", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 25,721", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 6,932", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:00.013 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 31 2024-09-13T14:19:00.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "1e646412-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPEDMD400G4_1e64e450-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141301": { 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:19:00.015 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "8DV101H0", 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e646412-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236781 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.016 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:00.017 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:19:00.018 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.019 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 27995823 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:00.020 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 49507808 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.023 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.024 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "63.999023" 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 89, 2024-09-13T14:19:00.025 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 3385, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 3399, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 3370 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 10, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 26, 2024-09-13T14:19:00.026 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 364889334, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 1834734314, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 6215216103, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 33084752516, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 11, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 56859, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 55867, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 25, 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 1468 2024-09-13T14:19:00.027 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 56859, 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 55867 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:00.028 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:00.029 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: 8DV101H0", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:00.030 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:00.031 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 1 - 512 8 2", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 2 - 512 16 2", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 3 - 4096 0 0", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 4 - 4096 8 0", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 5 - 4096 64 0", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: " 6 - 4096 128 0", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:00.032 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 25 Celsius", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 10%", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 11%", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 364,889,334 [186 TB]", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 1,834,734,314 [939 TB]", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 6,215,216,103", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 33,084,752,516", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 26", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 56,859", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 55,867", 2024-09-13T14:19:00.033 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 1,468", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:00.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 25 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.035 INFO:tasks.workunit.client.0.smithi026.stdout: "report": { 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer": { 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "active": true, 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "mode": "" 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "channels": [ 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "ident" 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "channels_available": [ 2024-09-13T14:19:00.036 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_available": [ 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:00.037 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_opted_in": [ 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:00.038 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "config": { 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "active_changed": [ 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:00.039 INFO:tasks.workunit.client.0.smithi026.stdout: "client_check_pool_perm", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_gid", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_uid", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "client_quota", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "daemonize", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "keyring", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_block_size", 2024-09-13T14:19:00.040 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_cache_size", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_compression", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_log", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_write_buffer_size", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_journald", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_stderr", 2024-09-13T14:19:00.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_host", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:00.042 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "no_config_file", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:00.043 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_lim", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_res", 2024-09-13T14:19:00.044 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addr", 2024-09-13T14:19:00.045 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addrv", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_default_features", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_qos_exclude_ops", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "setgroup", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "setuser" 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_changed": [ 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:00.046 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_join_fs", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/allow_ptrace", 2024-09-13T14:19:00.047 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/container_init", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/migration_current", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/use_repo_digest", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ssl_server_port", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/orchestrator/orchestrator", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_ident", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/description", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/organization", 2024-09-13T14:19:00.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:00.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:00.050 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:00.051 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_cache_enabled", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_ops_log", 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_usage_log" 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "contact": null, 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "crashes": [], 2024-09-13T14:19:00.052 INFO:tasks.workunit.client.0.smithi026.stdout: "created": "2024-09-13T14:09:50.810274Z", 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "crush": { 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_algs": { 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "straw2": 3 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_sizes": { 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "2": 1, 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "4": 2 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_types": { 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "1": 2, 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: "11": 1 2024-09-13T14:19:00.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "compat_weight_set": false, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "device_classes": [ 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: 8 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "num_buckets": 6, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "num_devices": 8, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "num_rules": 1, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "num_types": 12, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "num_weight_sets": 0, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "tunables": { 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "allowed_bucket_algs": 54, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_fallback_tries": 0, 2024-09-13T14:19:00.054 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_tries": 0, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_total_tries": 50, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_descend_once": 1, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_stable": 1, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_vary_r": 1, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v2_rules": 0, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v3_rules": 0, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v4_buckets": 1, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v5_rules": 0, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "legacy_tunables": 0, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "minimum_required_version": "jewel", 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "optimal_tunables": 1, 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "profile": "jewel", 2024-09-13T14:19:00.055 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables": 1, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables2": 1, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables3": 1, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables5": 1, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "straw_calc_version": 1 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "description": "upgrade test cluster", 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "fs": { 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "feature_flags": { 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "enable_multiple": true, 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_enabled_multiple": true 2024-09-13T14:19:00.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "filesystems": [ 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "approx_ctime": "2024-09", 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer_enabled": false, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_caps": 0, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_dns": 10, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_inos": 13, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_subtrees": 2, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_allowed_features": 0, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "explicitly_allowed_features": 0, 2024-09-13T14:19:00.057 INFO:tasks.workunit.client.0.smithi026.stdout: "files": 0, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "max_mds": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_data_pools": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_in": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_mds": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": 0, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_replay": 0, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_up": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "snaps": 0, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "standby_count_wanted": 0 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_mds": 1, 2024-09-13T14:19:00.058 INFO:tasks.workunit.client.0.smithi026.stdout: "total_num_mds": 2 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "hosts": { 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num": 2, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mds": 2, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mgr": 2, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mon": 2, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_osd": 2 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "leaderboard": false, 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "license": "sharing-1-0", 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "metadata": { 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:00.059 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 2 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:00.060 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 2 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 2 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 2 2024-09-13T14:19:00.061 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 3 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.062 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 3 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 3 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.063 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 3 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 8 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-13T14:19:00.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 8 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 8 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-13T14:19:00.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 8 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_objectstore": { 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": 8 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "rotational": { 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "0": 8 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:00.066 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "features": { 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "optional": [], 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "persistent": [ 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "kraken", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "luminous", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "mimic", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap-prune", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "nautilus", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "octopus", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "pacific", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "elector-pinging", 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: "quincy" 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:00.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv4_addr_mons": 6, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv6_addr_mons": 0, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "min_mon_release": 17, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "v1_addr_mons": 3, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "v2_addr_mons": 3 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "organization": "ceph-qa", 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_network": false, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "require_min_compat_client": "luminous", 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: "require_osd_release": "quincy" 2024-09-13T14:19:00.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": [ 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 1, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 1, 2024-09-13T14:19:00.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 1, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 1781760, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 1781760, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:00.070 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 126, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 236544, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 590368, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 590368, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 1771104, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 113, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 1667072 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:00.071 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 32, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 32, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 2, 2024-09-13T14:19:00.072 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 98304, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 98304, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 22, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:00.073 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 3410, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 3410, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 10230, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 51, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 19456 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:00.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 64, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 64, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 3, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:00.075 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 0.0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:00.076 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 0 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:00.077 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 8, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 8, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 4, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:00.078 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 12288, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 12288, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.689964257423071e-08, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:00.079 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 19, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 19, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 57, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 2, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 2048 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.080 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": { 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "mirroring_by_pool": [ 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: false 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_images_by_pool": [ 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pools": 1 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "report_id": "bb35b373-4dfb-4d9f-9cf9-b49d389b3568", 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "report_timestamp": "2024-09-13T14:18:59.976855", 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "report_version": 1, 2024-09-13T14:19:00.081 INFO:tasks.workunit.client.0.smithi026.stdout: "rook": { 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster": { 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "external-mode": null, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "allow-multiple-per-node": null, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "count": null, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "max-id": null, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "pvc": { 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "stretch": { 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.082 INFO:tasks.workunit.client.0.smithi026.stdout: "network": { 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "provider": null 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "storage": { 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "device-set": { 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "non-portable": null, 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "portable": null, 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.083 INFO:tasks.workunit.client.0.smithi026.stdout: "csi": { 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes": { 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "node": { 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes-total": null, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "with-ceph-daemons": null, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-cephfs-plugin": null, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-nfs-plugin": null, 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-rbd-plugin": null 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "storage-class": { 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket": null, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs": null, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "nfs": null, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": null, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:00.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "services": {}, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 105, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": 4, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_by_class": { 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "ssd": { 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_bytes": 2443300864, 2024-09-13T14:19:00.086 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_ratio": 0.003181545762345195 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:00.087 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:19:00.088 INFO:tasks.workunit.client.0.smithi026.stderr:++ ceph telemetry collection ls 2024-09-13T14:19:00.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:18:59 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:+ COLLECTIONS='New collections are available: 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.471 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore") 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.472 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.473 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.474 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.475 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:+ [[ New collections are available: 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr: 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:NAME STATUS DESC 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-13T14:19:00.479 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_options_bluestore* ]] 2024-09-13T14:19:00.480 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:00 smithi145 ceph-mon[31057]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-13T14:19:00.958 INFO:tasks.workunit.client.0.smithi026.stdout:Telemetry is on. 2024-09-13T14:19:00.958 INFO:tasks.workunit.client.0.smithi026.stdout:Some channels are disabled, please enable with: 2024-09-13T14:19:00.958 INFO:tasks.workunit.client.0.smithi026.stdout:`ceph telemetry enable channel perf` 2024-09-13T14:19:00.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[39182]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-mon[27770]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE Bus STOPPING 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE Bus STOPPED 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE Bus STARTING 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE Serving on http://:::9283 2024-09-13T14:19:00.967 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:00] ENGINE Bus STARTED 2024-09-13T14:19:00.969 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry enable channel perf 2024-09-13T14:19:01.433 INFO:tasks.workunit.client.0.smithi026.stdout:channel_perf is enabled 2024-09-13T14:19:01.444 INFO:tasks.workunit.client.0.smithi026.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE Bus STOPPING 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE Bus STOPPED 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE Bus STARTING 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE Serving on http://:::9283 2024-09-13T14:19:01.576 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:19:01] ENGINE Bus STARTED 2024-09-13T14:19:01.811 INFO:tasks.workunit.client.0.smithi026.stderr:+ LAST_OPT_REVISION=1 2024-09-13T14:19:01.812 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 1 -ne 1 ']' 2024-09-13T14:19:01.812 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry show 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:01.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:01.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:01 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:02.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:02.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:02.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:02.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:01 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:02.494 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:19:02.494 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer": { 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "active": true, 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "mode": "" 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "channels": [ 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "channels_available": [ 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:19:02.495 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_available": [ 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:02.496 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_opted_in": [ 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:02.497 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "config": { 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "active_changed": [ 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "client_check_pool_perm", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_gid", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_uid", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "client_quota", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "daemonize", 2024-09-13T14:19:02.498 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "keyring", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_block_size", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_cache_size", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_compression", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_log", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_write_buffer_size", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:02.499 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_journald", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_stderr", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_host", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:02.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "no_config_file", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:02.501 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-13T14:19:02.502 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_lim", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_res", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:02.503 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addr", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addrv", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_default_features", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_qos_exclude_ops", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "setgroup", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "setuser" 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_changed": [ 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:02.504 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:02.505 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_join_fs", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/allow_ptrace", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/container_init", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/migration_current", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/use_repo_digest", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ssl_server_port", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/orchestrator/orchestrator", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_ident", 2024-09-13T14:19:02.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_perf", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/description", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/enabled", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/organization", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:02.508 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:02.509 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_cache_enabled", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_ops_log", 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_usage_log" 2024-09-13T14:19:02.510 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "contact": null, 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "crashes": [], 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "created": "2024-09-13T14:09:50.810274Z", 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "crush": { 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_algs": { 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "straw2": 3 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_sizes": { 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "2": 1, 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "4": 2 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.511 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_types": { 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "1": 2, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "11": 1 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "compat_weight_set": false, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "device_classes": [ 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: 8 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_buckets": 6, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_devices": 8, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_rules": 1, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_types": 12, 2024-09-13T14:19:02.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_weight_sets": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "tunables": { 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "allowed_bucket_algs": 54, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_fallback_tries": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_tries": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_total_tries": 50, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_descend_once": 1, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_stable": 1, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_vary_r": 1, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v2_rules": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v3_rules": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v4_buckets": 1, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v5_rules": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "legacy_tunables": 0, 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "minimum_required_version": "jewel", 2024-09-13T14:19:02.513 INFO:tasks.workunit.client.0.smithi026.stdout: "optimal_tunables": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "profile": "jewel", 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables2": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables3": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables5": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "straw_calc_version": 1 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "description": "upgrade test cluster", 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "fs": { 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "feature_flags": { 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "enable_multiple": true, 2024-09-13T14:19:02.514 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_enabled_multiple": true 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "filesystems": [ 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "approx_ctime": "2024-09", 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer_enabled": false, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_caps": 0, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_dns": 10, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_inos": 13, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_subtrees": 2, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_allowed_features": 0, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "explicitly_allowed_features": 0, 2024-09-13T14:19:02.515 INFO:tasks.workunit.client.0.smithi026.stdout: "files": 0, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "max_mds": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_data_pools": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_in": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_mds": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": 0, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_replay": 0, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_up": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "snaps": 0, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "standby_count_wanted": 0 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_mds": 1, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "total_num_mds": 2 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.516 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_stats": { 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 20561920, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 444424, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 622592, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 431, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 1159088, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 20, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1233408, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 15103560, 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 21184512 2024-09-13T14:19:02.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 20135936, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 385160, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 786432, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 0, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 349, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 792488, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 12, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 853504, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 14696912, 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 20135936 2024-09-13T14:19:02.518 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 37453824, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1161800, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 425984, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 9011200, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1194, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 7411848, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 22, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1917696, 2024-09-13T14:19:02.519 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 23915056, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 46465024 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 35790848, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1092480, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 606208, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 9617408, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 990, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 6296088, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 21, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1702656, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 23471976, 2024-09-13T14:19:02.520 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 45408256 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 45056000, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1239784, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 344064, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 3399680, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1749, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 9885928, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 22, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 3096576, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 27868208, 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 48455680 2024-09-13T14:19:02.521 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 274079744, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 862448, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 909312, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1400, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5263048, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2701056, 2024-09-13T14:19:02.522 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262500680, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 274989056 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 272711680, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 780328, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 950272, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1348, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5052304, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 46, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2555648, 2024-09-13T14:19:02.523 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261570888, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273661952 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 307789824, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 833688, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 4849664, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 753664, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1245, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4993880, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:02.524 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1924608, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 295187984, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 308543488 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271785984, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 769328, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 827392, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1261, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4637992, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1760000, 2024-09-13T14:19:02.525 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261866152, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 272613376 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271335424, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 758384, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 163840, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1236, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4937048, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 46, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1660416, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261227064, 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 271499264 2024-09-13T14:19:02.526 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 273154048, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 734824, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 8192, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 786432, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1282, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4994944, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2304000, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262359576, 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273940480 2024-09-13T14:19:02.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271785984, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 772456, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 827392, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1256, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4962832, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2010880, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261287304, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 272613376 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.528 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 272818176, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 969616, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 24576, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 909312, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1316, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4663080, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:02.529 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2339584, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262068808, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273727488 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "hosts": { 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num": 2, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mds": 2, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mgr": 2, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mon": 2, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_osd": 2 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "io_rate": { 2024-09-13T14:19:02.530 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_stats_delta": { 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "stamp_delta": "12.002607", 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:02.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:02.532 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:02.533 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:02.534 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 0, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 0 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "leaderboard": false, 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "license": "sharing-1-0", 2024-09-13T14:19:02.535 INFO:tasks.workunit.client.0.smithi026.stdout: "mempool": { 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.536 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.538 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.539 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.540 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 222471, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 72 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.541 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 50488, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 208 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8136, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 139 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.543 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.544 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.545 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.546 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.547 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.549 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 150088, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 33 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 856, 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 2 2024-09-13T14:19:02.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.551 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.553 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.554 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.555 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.556 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.557 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.558 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1293574, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 406 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.559 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:02.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5312, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 103 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.561 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.562 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.563 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.564 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.565 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.566 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1092166, 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 356 2024-09-13T14:19:02.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.568 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:02.569 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5312, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 103 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.571 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.573 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.574 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1136742, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 353 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 352, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.576 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10624, 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 206 2024-09-13T14:19:02.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.580 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20384, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 672, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 6 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8736, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20384, 2024-09-13T14:19:02.581 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5680, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 71 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 38022, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 54150, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9794 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.582 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 133960, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10972, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 708 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 31244, 2024-09-13T14:19:02.583 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 409193, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 90 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.584 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2449096, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5896, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 67 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 509696, 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 44 2024-09-13T14:19:02.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 69472, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 136 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.586 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.587 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.588 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 896, 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8112, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4640, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 58 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.590 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19182, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48126, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8781 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 124440, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9132, 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 533 2024-09-13T14:19:02.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27224, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 736, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.592 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 410401, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 90 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12096, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2388365, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4928, 2024-09-13T14:19:02.593 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 56 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 440192, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 38 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.594 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8528, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 22 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236944, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3552 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 112, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8112, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.598 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4160, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 52 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4096, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 46328, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8543 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.599 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 121040, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 178 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8332, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 525 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 25944, 2024-09-13T14:19:02.600 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 178 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 321941, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 68 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.601 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2405328, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 176 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4224, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 48 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.602 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 359104, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 31 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10400, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 28 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236688, 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3540 2024-09-13T14:19:02.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.604 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.605 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 448, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8304, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3840, 2024-09-13T14:19:02.607 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 48 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 15086, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 47276, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8775 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 124440, 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:02.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9132, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 26552, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.609 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2208, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 298663, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 63 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30637, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2444010, 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:02.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4928, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 56 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 428608, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 37 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.611 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10400, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 20 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.612 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:02.613 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 672, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 6 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8304, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:02.615 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3520, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 44 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4096, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48382, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8923 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.616 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 126480, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9324, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27320, 2024-09-13T14:19:02.617 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 736, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 913585, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 69 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12096, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.618 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2397418, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4840, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 55 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 451776, 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 39 2024-09-13T14:19:02.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8944, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 18 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236432, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3528 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.620 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.621 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.622 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19152, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 224, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 2 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.623 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8208, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19152, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3040, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 38 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.624 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 49086, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8963 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 127160, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9540, 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:02.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27912, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2208, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 838740, 2024-09-13T14:19:02.626 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 51 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30637, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2417468, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 181 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5456, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 62 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 474944, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 41 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 13520, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 26 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.628 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236944, 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3552 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:02.630 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.631 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20160, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8640, 2024-09-13T14:19:02.632 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20160, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2720, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 34 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.633 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48697, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9204 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 125800, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 185 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10564, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 699 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.634 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 28428, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 185 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 497356, 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 47 2024-09-13T14:19:02.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2403892, 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 167 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.636 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4576, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 52 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 451776, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 39 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.637 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 69472, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 144 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.638 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.639 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:02.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20048, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1232, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 11 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8592, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20048, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.641 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2160, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 27 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 293272, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 12 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 51658, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9593 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 131240, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 193 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.642 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 11328, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 704 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30644, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 193 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 178783, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 35 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2420703, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 226 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:02.644 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6248, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 71 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 532864, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 46 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 69992, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.646 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "metadata": { 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:02.647 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 2 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 2 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:02.648 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 2 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 2 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:02.649 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 3 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 3 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 3 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.650 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 3 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 8 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:02.651 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 8 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 8 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-13T14:19:02.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 8 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_objectstore": { 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": 8 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "rotational": { 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: "0": 8 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.653 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "features": { 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "optional": [], 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "persistent": [ 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "kraken", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "luminous", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "mimic", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap-prune", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "nautilus", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "octopus", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "pacific", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "elector-pinging", 2024-09-13T14:19:02.654 INFO:tasks.workunit.client.0.smithi026.stdout: "quincy" 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv4_addr_mons": 6, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv6_addr_mons": 0, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "min_mon_release": 17, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "v1_addr_mons": 3, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "v2_addr_mons": 3 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "organization": "ceph-qa", 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_network": false, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "require_min_compat_client": "luminous", 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: "require_osd_release": "quincy" 2024-09-13T14:19:02.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_perf_histograms": [ 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:02.656 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:02.657 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:02.659 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:02.660 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.661 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.662 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.662 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.662 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:02.663 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.663 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.664 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.665 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.666 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.667 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.668 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.669 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.670 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.671 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.672 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.673 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.674 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.675 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.676 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.677 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.678 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.679 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.680 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.681 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:02.682 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.683 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.684 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:02.685 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:02.686 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:02.687 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.688 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:02.689 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:02.690 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.691 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.692 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.693 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.694 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.695 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.697 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.698 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.699 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.700 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.701 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.702 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.703 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.704 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.705 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.706 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.707 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.708 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.709 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.710 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.711 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:02.713 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:02.714 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:02.716 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:02.717 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.718 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.719 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.720 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.721 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.722 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.723 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.724 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.725 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:02.726 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.727 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.728 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.729 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.730 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.731 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.732 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.733 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.734 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.735 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.736 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.738 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.739 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.740 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:02.741 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:02.742 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:02.743 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:02.744 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:02.745 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.746 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.747 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.748 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 5, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 16, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.749 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.750 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:02.751 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.752 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.753 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.754 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.755 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.756 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.757 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.758 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.759 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.760 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.761 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.762 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.763 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.764 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:02.765 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.766 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.767 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_counters": { 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:02.768 INFO:tasks.workunit.client.0.smithi026.stdout: "caps": { 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_flush_ack": { 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_grant": { 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_revoke": { 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_trunc": { 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_commit": { 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_fetch": { 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_merge": { 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_split": { 2024-09-13T14:19:02.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "exported_inodes": { 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "forward": { 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_cap_release": { 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_caps": { 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.771 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_caps_dirty": { 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_inode_file_caps": { 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "imported_inodes": { 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes": { 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_expired": { 2024-09-13T14:19:02.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_pinned": { 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_with_caps": { 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "load_cent": { 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "openino_dir_fetch": { 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "process_request_cap_release": { 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "reply_latency": { 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 80402 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "request": { 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rbytes": { 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rfiles": { 2024-09-13T14:19:02.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rsnaps": { 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_reply": { 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "subtrees": { 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_cache": { 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_enqueue_scrub": { 2024-09-13T14:19:02.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_exportdir": { 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_flush": { 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_fragmentdir": { 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_fragstats": { 2024-09-13T14:19:02.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_inodestats": { 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_enqueued": { 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_prioritized": { 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_processing": { 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays": { 2024-09-13T14:19:02.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays_delayed": { 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays_enqueuing": { 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_completed": { 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_started": { 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_created": { 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_enqueued": { 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_migrated": { 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_reintegrated": { 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_log": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "ev": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "evadd": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "evex": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "evexd": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "evexg": { 2024-09-13T14:19:02.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "evtrm": { 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "jlat": { 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 11659984 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "replayed": { 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "seg": { 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "segadd": { 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "segex": { 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "segexd": { 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "segexg": { 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "segtrm": { 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.782 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_mem": { 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "cap": { 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "cap+": { 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "cap-": { 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "dir": { 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.783 INFO:tasks.workunit.client.0.smithi026.stdout: "dir+": { 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "dir-": { 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "dn": { 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "dn+": { 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "dn-": { 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.784 INFO:tasks.workunit.client.0.smithi026.stdout: "heap": { 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 207068 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "ino": { 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "ino+": { 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "ino-": { 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "rss": { 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38944 2024-09-13T14:19:02.785 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_server": { 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "cap_acquisition_throttle": { 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "cap_revoke_eviction": { 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_request": { 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_session": { 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.786 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_peer_request": { 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "req_create_latency": { 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getattr_latency": { 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 80402 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getfilelock_latency": { 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getvxattr_latency": { 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "req_link_latency": { 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookup_latency": { 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookuphash_latency": { 2024-09-13T14:19:02.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupino_latency": { 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupname_latency": { 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupparent_latency": { 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.789 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupsnap_latency": { 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lssnap_latency": { 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mkdir_latency": { 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mknod_latency": { 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mksnap_latency": { 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "req_open_latency": { 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "req_readdir_latency": { 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rename_latency": { 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "req_renamesnap_latency": { 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmdir_latency": { 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.792 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmsnap_latency": { 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmxattr_latency": { 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setattr_latency": { 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setdirlayout_latency": { 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setfilelock_latency": { 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setlayout_latency": { 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setxattr_latency": { 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.794 INFO:tasks.workunit.client.0.smithi026.stdout: "req_symlink_latency": { 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "req_unlink_latency": { 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_sessions": { 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "average_load": { 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "avg_session_uptime": { 2024-09-13T14:19:02.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 89 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "mdthresh_evicted": { 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "session_count": { 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "session_remove": { 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.796 INFO:tasks.workunit.client.0.smithi026.stdout: "sessions_open": { 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "sessions_stale": { 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "total_load": { 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.797 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "purge_queue": { 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executed": { 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.798 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing": { 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_high_water": { 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_ops": { 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_ops_high_water": { 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_item_in_journal": { 2024-09-13T14:19:02.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 52 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:02.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 824 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 824, 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3920293 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:02.802 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 824, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 128997060 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 146 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 999977 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:02.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 825 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:02.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 825 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.806 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 825, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1882957106 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:02.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 825 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 825, 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7685839 2024-09-13T14:19:02.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 825, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5547 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 825, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 379991355 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.809 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 43843584 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 33857536 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9986048 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021340 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271552 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70834340 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021340 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41237 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.815 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71064655 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.816 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184680893 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71171651 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.819 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26290536 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 35610, 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 80470354 2024-09-13T14:19:02.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1797, 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 603827144 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 11 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:02.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 74 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:02.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 815 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 815, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3894623 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 815, 2024-09-13T14:19:02.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 79998816 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:02.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 817 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.827 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:02.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 817 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 817, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1845972656 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:02.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:02.830 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 816 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 816, 2024-09-13T14:19:02.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7636408 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 816, 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5476 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 816, 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 115998284 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:02.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42786816 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31809536 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10977280 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021755 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270445 2024-09-13T14:19:02.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.835 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70835032 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021755 2024-09-13T14:19:02.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41237 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71064240 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.840 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184681170 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.842 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71171374 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 30482974 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 34803, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 87420789 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.844 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1751, 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 217297549 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.846 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:02.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 176 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 948 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:02.848 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4968545 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2844 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 160997617 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.849 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:02.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 948 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4900347 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5399 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:02.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 948, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 138149606 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 999985 2024-09-13T14:19:02.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 950 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 950, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2285966154 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.853 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1627 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.854 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45834240 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41246720 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4587520 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277022402 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271122 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70833708 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277022402 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38834 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71065996 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184681601 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71170943 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:02.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31667556 2024-09-13T14:19:02.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.867 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:02.867 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 42226, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 169335728 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.869 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2063, 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 280705837 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.871 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2297856 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:02.872 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1544192 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1404928 2024-09-13T14:19:02.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:02.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:02.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:02.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3776512 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:02.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1298432 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 239524 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:02.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 35705 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:02.881 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 326, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70191980 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 318, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9800935 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 326, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 430979 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 326, 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70622959 2024-09-13T14:19:02.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32588 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7934 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10163 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 692 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:02.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 218 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 114, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1416171 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 272, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 238478 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 158, 2024-09-13T14:19:02.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 793004 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55136 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6342914 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:02.886 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 99, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7152809 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 98, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 414612138315 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 99, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 150024659481 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 424, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 76015136197 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.887 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 424, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50256 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 415794 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 141976462 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 168597 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 86972384 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19932307 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 901055 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:02.889 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 59, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 267573 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 255725325 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23913414 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 425, 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4026285 2024-09-13T14:19:02.890 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 272236544 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271253504 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:02.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 983040 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85553054 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24576 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73805922 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641647910 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.896 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71381642 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.898 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57035369 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68793287 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641647910 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 278461 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71105309 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:02.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.905 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 604 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46338348 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3838826 2024-09-13T14:19:02.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26641510 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.910 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.911 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 604 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46338348 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3838826 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.912 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26641510 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 570 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:02.913 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305790976 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:02.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2064384 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:02.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1400832 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1261568 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.917 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:02.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:02.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:02.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.921 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.922 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3399680 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 692224 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.923 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 234021 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 24, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60822 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:02.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55445273 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 286, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9453820 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 431815 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55877088 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.926 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31093 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14142 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.927 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13337 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 683 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 223 2024-09-13T14:19:02.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 116, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1837745 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 285479 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 155, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1178382 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.929 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 110282 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6930128 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 95, 2024-09-13T14:19:02.930 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7180933 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 94, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 380165209420 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 95, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 120202842339 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 365, 2024-09-13T14:19:02.931 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 86309478108 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 365, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 47205 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 391418 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 97503526 2024-09-13T14:19:02.932 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 128312 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 59983516 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23209152 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:02.933 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 309520 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 54, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 309689 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 188077449 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 27536659 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:02.934 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4367024 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269860864 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269819904 2024-09-13T14:19:02.935 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40960 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85555323 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.937 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73815941 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.939 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641664929 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71364623 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:02.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57036882 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.943 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791774 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.945 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641664929 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.946 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.947 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 252990 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71113761 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.948 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 97 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.950 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1640827 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 411266444 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 602 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:02.951 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 34402695 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 121526526 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 18 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2687287 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.952 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 602 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2243773 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1806843 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.953 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10415687 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:02.954 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4490655 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9714590 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.955 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1640827 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 398163470 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:02.956 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 27668267 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 110005093 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:02.957 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305184768 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.958 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1716224 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.960 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1159168 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1019904 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:02.961 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:02.962 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:02.963 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.964 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:02.965 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.966 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2809856 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.967 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 692224 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 220296 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.968 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9595 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.969 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50371502 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 227, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9769353 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 351683 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:02.970 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50723185 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42896 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 942 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.971 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7732 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 500 2024-09-13T14:19:02.972 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 215 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 861316 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 214, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 262146 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:02.973 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 123, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 420533 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 101916 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8444398 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.974 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8036179 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 347940565486 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 109038186035 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.975 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 76361559862 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32640 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 583496 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 104043309 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.976 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 114195 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 75985560 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15925543 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 310541 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.977 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 48, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 233233 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 205042362 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 21108256 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2204795 2024-09-13T14:19:02.978 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 303693824 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 302841856 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:02.979 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 851968 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.980 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85556299 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.981 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73823157 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.983 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641672243 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.984 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71357309 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:02.986 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57037532 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.987 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.988 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791124 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:02.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641672243 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 247723 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.991 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71111714 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.992 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31 2024-09-13T14:19:02.993 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.994 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 91874897 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5476516 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:02.995 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42709457 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:02.996 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:02.998 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:02.999 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 91874897 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5476516 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42709457 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.000 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305180672 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.001 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:03.002 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1122304 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.003 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.004 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 851968 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 712704 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.005 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.006 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:03.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.008 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.009 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:03.010 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1908736 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 708608 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:03.011 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:03.012 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7317 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40513826 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 150, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6393392 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.013 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 239914 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40753740 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.014 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45696 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1024 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:03.015 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9061 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 343 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 215 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 820702 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.016 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 160, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 248326 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 69, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 222833 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.017 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3323354 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3738964 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 47, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 291757300607 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:03.018 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 75206048061 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 234, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 577313 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 234, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26764 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 280409 2024-09-13T14:19:03.019 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 138019317 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68458 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71471481 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:03.020 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13213499 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312699 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 189006 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 226335230 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.021 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16663999 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1908882 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.022 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268812288 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268738560 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73728 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:03.023 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85555622 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.024 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.025 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8192 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73819738 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.026 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641667169 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.027 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.028 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.029 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71362383 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.030 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57037081 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.031 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.032 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791575 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.033 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641667169 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.034 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6964 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 247939 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.035 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71109608 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.036 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 37 2024-09-13T14:19:03.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:03.038 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1148904 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:03.039 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:03.040 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:03.041 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.042 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:03.043 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1148904 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.044 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305197056 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.046 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:03.047 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1323008 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.048 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1011712 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 872448 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:03.049 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.050 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:03.051 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:03.052 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.053 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.054 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2269184 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 708608 2024-09-13T14:19:03.055 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 302921 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.056 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5538 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44558699 2024-09-13T14:19:03.057 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5843060 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 225243 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44783942 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:03.058 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29553 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1061 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9990 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 376 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 231 2024-09-13T14:19:03.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 93, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 941978 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195067 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 64, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 321136 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:03.061 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 108690 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3783863 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 42, 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2954084 2024-09-13T14:19:03.062 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 212467068807 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 42, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68124457548 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 273, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8605932315 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:03.063 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 273, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31816 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 330120 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 81533165 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 87630 2024-09-13T14:19:03.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 52756757 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13780348 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 310686 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 40, 2024-09-13T14:19:03.065 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 196471 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 152222169 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16713138 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2274789 2024-09-13T14:19:03.066 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268746752 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268402688 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:03.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 344064 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.068 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498177 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.069 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.070 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.070 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.070 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.070 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.071 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.072 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73881279 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.073 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236333 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.074 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.075 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71793219 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.076 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998785 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.077 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.078 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829871 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.079 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236333 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 253659 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71541688 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39 2024-09-13T14:19:03.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:03.084 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 22 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4942962 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1022875 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.085 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4845855 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.086 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.087 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.088 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.089 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 22 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4942962 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1022875 2024-09-13T14:19:03.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4845855 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.091 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305201152 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:03.092 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.093 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1167360 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.094 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 868352 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 729088 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:03.095 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.096 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:03.098 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.099 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1970176 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.101 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 700416 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 248067 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 37035 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.103 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60565413 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 160, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5433846 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 174038 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:03.104 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60739451 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 33837 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5874 2024-09-13T14:19:03.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4482 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:03.106 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 354 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 238 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1728477 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 147, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 273499 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:03.107 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 56, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1352842 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 78993 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3510043 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.108 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2918961 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 38, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 276986572867 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65698492633 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8286324816 2024-09-13T14:19:03.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 28134 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 261894 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 119335641 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:03.110 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 82830 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 78836259 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32156842 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312074 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.111 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 33, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 176420 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 234138535 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 34407909 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:03.112 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2866983 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271187968 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270254080 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:03.113 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 933888 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.114 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498351 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.115 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.116 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73885201 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.117 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641237637 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.118 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.119 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71791915 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.120 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998901 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.121 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.122 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829755 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.123 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641237637 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.124 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6963 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 253470 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.125 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71533610 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.126 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:03.127 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.128 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.129 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:03.130 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.131 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:03.132 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:03.133 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.134 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.135 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305188864 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.136 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:03.137 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1200128 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.138 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 954368 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 815104 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:03.140 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:03.141 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.142 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:03.144 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2088960 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1290240 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:03.145 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:03.146 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19931 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 182, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 53512531 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:03.147 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 172, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5710882 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 182, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 170392 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 182, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 53682923 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 33119 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4656 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:03.149 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9111 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 313 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 240 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:03.150 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 89, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1170337 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 141, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 183675 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 52, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 943629 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:03.151 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2275115 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2510008 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 33, 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 229090981017 2024-09-13T14:19:03.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 53719950822 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 468598 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 27171 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:03.153 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 196697 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 93326717 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 81171 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 53801464 2024-09-13T14:19:03.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 11986193 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 902669 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 29, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 135244 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 161624383 2024-09-13T14:19:03.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14216313 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 240, 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2438134 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:03.156 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269860864 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268861440 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 999424 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:03.157 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498208 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.158 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.159 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73885344 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236564 2024-09-13T14:19:03.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.162 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71792988 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:03.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998805 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.165 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829851 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.167 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236564 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7103 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 255713 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.169 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71532300 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.171 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2804 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42228473 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 15, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7772952 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29658765 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:03.173 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:03.174 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.175 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:03.176 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2804 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42228473 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.177 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 15, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7772952 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29658765 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.178 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305778688 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:03.179 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.180 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1142784 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 909312 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 770048 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.182 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:03.183 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.184 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:03.185 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.186 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.187 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1986560 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1286144 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:03.188 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 216725 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:03.189 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 17025 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 171, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 47253947 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:03.190 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4589516 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 171, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 116903 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 171, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 47370850 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.191 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 30608 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1102 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.192 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4540 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 368 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 258 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 92, 2024-09-13T14:19:03.193 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128607 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 137, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 276731 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 45, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1536263 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:03.194 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 61887 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1936817 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 23, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1715724 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 22, 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 132839797878 2024-09-13T14:19:03.195 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 23, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39433865849 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 277, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 434562 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 277, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 30586 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:03.196 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 188102 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 123236546 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 84112 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70539539 2024-09-13T14:19:03.197 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12199803 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 900361 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 22, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 99790 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 208135508 2024-09-13T14:19:03.198 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14570414 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 278, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1947624 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.199 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270974976 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269910016 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1064960 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.200 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85488887 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.201 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 143360 2024-09-13T14:19:03.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73751305 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.203 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641166657 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.204 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.205 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71862895 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.206 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56992591 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.207 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.209 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68836065 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641166657 2024-09-13T14:19:03.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:03.211 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271958 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.212 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71593065 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.213 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.214 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 29, 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 49880993 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:03.215 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 131762 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 31, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7780832 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 29, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26552229 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 18 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:03.216 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2983563 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 131762 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2673077 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:03.217 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2309843 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4751505 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2089923 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4507330 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.219 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42145925 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:03.220 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3017832 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19735056 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:03.221 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305778688 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": [ 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:03.222 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 1, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 1, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 1, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 1781760, 2024-09-13T14:19:03.223 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 1781760, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 126, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 236544, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 590368, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 590368, 2024-09-13T14:19:03.224 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 1771104, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 113, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 1667072 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:03.225 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 32, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 32, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 2, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 98304, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 98304, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:03.226 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 22, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 3410, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 3410, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 10230, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 51, 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 19456 2024-09-13T14:19:03.227 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:03.228 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 64, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 64, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 3, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:03.229 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 0.0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 0 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:03.230 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 8, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 8, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 4, 2024-09-13T14:19:03.231 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115933962, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 12288, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 12288, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.689964257423071e-08, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:03.232 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 19, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 19, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 57, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 2, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 2048 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.233 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": { 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "mirroring_by_pool": [ 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: false 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "num_images_by_pool": [ 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pools": 1 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "report_id": "bb35b373-4dfb-4d9f-9cf9-b49d389b3568", 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "report_timestamp": "2024-09-13T14:19:02.363260", 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "report_version": 1, 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_stats": { 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "6.15.5" 2024-09-13T14:19:03.234 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "rook": { 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster": { 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "external-mode": null, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "allow-multiple-per-node": null, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "count": null, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "max-id": null, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "pvc": { 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "stretch": { 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "network": { 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "provider": null 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "storage": { 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "device-set": { 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "non-portable": null, 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "portable": null, 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "csi": { 2024-09-13T14:19:03.236 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes": { 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "node": { 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes-total": null, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "with-ceph-daemons": null, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-cephfs-plugin": null, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-nfs-plugin": null, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-rbd-plugin": null 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.237 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "storage-class": { 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket": null, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs": null, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "nfs": null, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": null, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "services": {}, 2024-09-13T14:19:03.238 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_per_pg": [ 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.239 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568075+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568075+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568393+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.241 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3f", 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000", 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.242 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.245 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.246 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563364+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563364+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.563624+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.247 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.248 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3e", 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.249 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000", 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.253 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566331+0000", 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566331+0000", 2024-09-13T14:19:03.254 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566586+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:03.255 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3d", 2024-09-13T14:19:03.256 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000", 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.257 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.260 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.261 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.564496+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.564496+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564769+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.263 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3c", 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.264 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000", 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.267 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.268 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555311+0000", 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555311+0000", 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555592+0000", 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.270 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3b", 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.271 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000", 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.276 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.277 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571310+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571310+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572204+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.279 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3a", 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000", 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.280 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.283 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.284 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566403+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566403+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566676+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.286 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.39", 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000", 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.290 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.291 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566400+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566400+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566663+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.293 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.38", 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000", 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.294 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.297 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.298 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.569776+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.569776+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570338+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.300 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.37", 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000", 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.301 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.304 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.305 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563959+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563959+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564796+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.306 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.307 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.36", 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.308 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000", 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.309 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.312 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.313 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566726+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566726+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567040+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.314 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.315 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.35", 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000", 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.316 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.317 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.318 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.319 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.320 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563402+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563402+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.563679+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.321 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.322 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.34", 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000", 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.323 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.324 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.325 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.326 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.327 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.552354+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.552354+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.552634+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.328 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.329 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.33", 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000", 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.330 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.331 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.332 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.333 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.334 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.575737+0000", 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.575737+0000", 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.576136+0000", 2024-09-13T14:19:03.335 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.336 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.32", 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.337 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000", 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.338 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.339 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.340 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.341 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.342 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565645+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565645+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566237+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.343 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.344 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.31", 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000", 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.346 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.347 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.348 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.349 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.964708+0000", 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.964708+0000", 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.965107+0000", 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.350 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.351 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.30", 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.352 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000", 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.354 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.355 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.356 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565091+0000", 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565091+0000", 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565535+0000", 2024-09-13T14:19:03.357 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:03.358 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2f", 2024-09-13T14:19:03.359 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000", 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.362 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.363 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.364 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.556349+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.556349+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.556686+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.365 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.366 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2e", 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000", 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.370 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.371 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563703+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563703+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564825+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.373 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2d", 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000", 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.377 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.378 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568070+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568070+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568327+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.380 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2c", 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000", 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.381 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.384 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.385 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568056+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568056+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568339+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.387 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2b", 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000", 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.388 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.391 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.392 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572126+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572126+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572417+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.393 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.394 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2a", 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.395 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000", 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.396 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.399 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.400 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555172+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555172+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555819+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.402 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.29", 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000", 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.403 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.406 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.559771+0000", 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.559771+0000", 2024-09-13T14:19:03.407 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.561070+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.408 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.409 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.28", 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000", 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.410 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.411 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.413 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.414 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.553231+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.553231+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.554841+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.415 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.416 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.27", 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000", 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.417 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.418 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.420 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.421 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.550739+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.550739+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.551060+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.422 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.423 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.26", 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000", 2024-09-13T14:19:03.424 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.425 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.427 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.428 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.559401+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.559401+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.559670+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.429 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.430 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.25", 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.431 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000", 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.432 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.433 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.435 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.436 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570531+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570531+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570832+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.437 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.438 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.24", 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.439 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000", 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.440 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.442 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.443 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572783+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572783+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573053+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.444 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.445 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.23", 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000", 2024-09-13T14:19:03.446 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.447 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.448 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.449 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.450 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568939+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568939+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.569346+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.451 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.452 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.22", 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000", 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.453 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.454 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.455 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.456 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.457 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562267+0000", 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562267+0000", 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562543+0000", 2024-09-13T14:19:03.458 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:03.459 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.21", 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.460 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000", 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.461 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.462 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.464 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.465 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568169+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568169+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568440+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.466 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.467 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.20", 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.468 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000", 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.469 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.470 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.472 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963029+0000", 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963029+0000", 2024-09-13T14:19:03.473 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963486+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.474 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1a", 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.475 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000", 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.476 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.480 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.481 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.539574+0000", 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.539574+0000", 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541205+0000", 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.482 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.483 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1b", 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.484 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000", 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.488 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.489 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.605583+0000", 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.605583+0000", 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.606065+0000", 2024-09-13T14:19:03.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:03.491 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.492 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1b", 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:14:21.907855+0000", 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.497 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543138+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543138+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.543444+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.499 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.500 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1a", 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000", 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.504 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:03.505 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.539753+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.539753+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541457+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.507 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.508 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.19", 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000", 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.509 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.511 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.512 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.513 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.514 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570884+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570884+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571137+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.515 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.516 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.18", 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000", 2024-09-13T14:19:03.517 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.518 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.519 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.520 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.521 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.522 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545863+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545863+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546875+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.523 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.524 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.18", 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.525 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000", 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.526 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.528 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.529 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.530 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.560291+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.560291+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.560575+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.531 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.532 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.19", 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000", 2024-09-13T14:19:03.533 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.534 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.535 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.538 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545833+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545833+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546137+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.539 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.540 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.17", 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000", 2024-09-13T14:19:03.541 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.542 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.545 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.546 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565363+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565363+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566451+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.548 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.16", 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.549 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000", 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.553 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.554 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540611+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540611+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.540966+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.556 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.16", 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.557 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000", 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.561 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.562 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.573138+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.573138+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573426+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.17", 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.565 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000", 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.568 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.569 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.570 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542739+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542739+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.543031+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.571 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 3, 2024-09-13T14:19:03.572 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 3, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.15", 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.573 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 27, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000", 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 90, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:03.575 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.576 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 4, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.577 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'3" 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.578 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.550913+0000", 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.550913+0000", 2024-09-13T14:19:03.579 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.551575+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.580 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.581 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.14", 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000", 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.582 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.583 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.584 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.586 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.587 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.538864+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.538864+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.539191+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.588 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 12, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.589 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 12, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.14", 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 36, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000", 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.590 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 2624, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.591 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 12, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 4, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 4, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.592 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 12, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 9 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.594 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "67'12" 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.595 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.567513+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.567513+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567770+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.596 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.597 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.15", 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.598 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000", 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.599 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.602 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.603 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545135+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545135+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545520+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.604 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.605 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.13", 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 26, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000", 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.606 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 34, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.607 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 6 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:03.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.611 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.574648+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.574648+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.575023+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.613 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.12", 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000", 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.614 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.617 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.618 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545255+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545255+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546988+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.619 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.620 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.12", 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.621 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000", 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.625 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.626 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555391+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555391+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555682+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.13", 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.629 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000", 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.633 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.634 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.541726+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.541726+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542050+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.636 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.11", 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.637 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000", 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.641 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.642 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566924+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566924+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567258+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.644 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.10", 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000", 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.649 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.650 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540793+0000", 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540793+0000", 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541058+0000", 2024-09-13T14:19:03.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:03.652 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 8, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 8, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.653 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.10", 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000", 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 12, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 4, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 4, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 4, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.656 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 12, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.657 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'8" 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.658 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562669+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562669+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562994+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.659 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.660 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.11", 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.661 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000", 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.663 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.664 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.665 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.666 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540386+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540386+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.540681+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.667 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.668 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.f", 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000", 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.669 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 22, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.671 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.672 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.673 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.558987+0000", 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.558987+0000", 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.559281+0000", 2024-09-13T14:19:03.674 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.675 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.676 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.e", 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000", 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.677 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.678 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.679 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.680 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.681 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543782+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543782+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.544147+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.682 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.683 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.e", 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.684 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000", 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.685 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.686 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.687 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.688 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.689 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.569357+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.569357+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570603+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.690 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.691 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.f", 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.692 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000", 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.693 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.694 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.696 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.697 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.544417+0000", 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.544417+0000", 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546363+0000", 2024-09-13T14:19:03.698 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:03.699 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.700 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.d", 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000", 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 70, 2024-09-13T14:19:03.701 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:03.704 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.705 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572166+0000", 2024-09-13T14:19:03.706 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572166+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572445+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:03.707 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.708 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.c", 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000", 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.712 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.713 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542641+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542641+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542920+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.715 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.c", 2024-09-13T14:19:03.716 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000", 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.720 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.721 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571003+0000", 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571003+0000", 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572195+0000", 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.722 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.723 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.d", 2024-09-13T14:19:03.724 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000", 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.729 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.376845+0000", 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.376845+0000", 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426886+0000", 2024-09-13T14:19:03.730 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:03.731 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.732 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.5", 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:07:10.386795+0000", 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000554698, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.733 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.736 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.737 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 19, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:14:57.475298+0000", 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:14:57.475298+0000", 2024-09-13T14:19:03.738 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:14:57.479980+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 50, 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:03.739 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427505+0000", 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 79, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 49, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 79, 2024-09-13T14:19:03.740 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "1.0", 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 57, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000", 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.741 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 590368, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 590368, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.742 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.743 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 126, 2024-09-13T14:19:03.744 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 231, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 113, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1628 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.745 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "60'79" 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.746 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.541756+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.541756+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542042+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.747 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.748 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.3", 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.749 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000", 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.750 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.751 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.752 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.753 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.754 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565213+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565213+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565510+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.755 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.756 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.757 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2", 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000", 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.758 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.759 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.760 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.761 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.762 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:22.202033+0000", 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:22.202033+0000", 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426290+0000", 2024-09-13T14:19:03.763 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.764 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.765 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.6", 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:15:48.139777+0000", 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000710344, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.766 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.767 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.768 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.769 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.770 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549995+0000", 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549995+0000", 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550379+0000", 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.771 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.772 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.0", 2024-09-13T14:19:03.773 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000", 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.774 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.775 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.776 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.777 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.778 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.964646+0000", 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.964646+0000", 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.965011+0000", 2024-09-13T14:19:03.779 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:03.780 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.781 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1", 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:36:07.136822+0000", 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.783 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.784 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.785 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.786 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605160+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605160+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403479+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.787 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.788 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.7", 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.789 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:55:48.959952+0000", 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000342076, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.790 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.791 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.792 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.793 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.794 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543880+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543880+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.544154+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.795 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.796 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1", 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.797 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000", 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.798 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.800 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.802 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.552054+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.552054+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.553134+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.803 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.804 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.0", 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000", 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.805 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.806 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.807 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.809 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:22.202029+0000", 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:22.202029+0000", 2024-09-13T14:19:03.810 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403217+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.811 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.4", 2024-09-13T14:19:03.812 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:09:48.284490+0000", 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000388303, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.813 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.814 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.815 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.816 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.817 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545610+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545610+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545901+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.818 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.819 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.2", 2024-09-13T14:19:03.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000", 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.821 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.822 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.823 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.824 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.825 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562679+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562679+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562994+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.826 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.827 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3", 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000", 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.828 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.829 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.830 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.831 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.832 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605236+0000", 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605236+0000", 2024-09-13T14:19:03.833 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403491+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.834 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:03.835 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.2", 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:10:47.051775+0000", 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000975641, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.836 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 19, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.837 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.838 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 2, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 2 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.839 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "66'2" 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.840 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549305+0000", 2024-09-13T14:19:03.841 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549305+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549591+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.842 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.843 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.4", 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000", 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.844 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.845 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.846 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.847 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.848 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.561391+0000", 2024-09-13T14:19:03.849 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.561391+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562976+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.850 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.851 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.5", 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000", 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.852 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.853 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.854 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.855 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.857 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.604772+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.604772+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426585+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.858 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.859 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.3", 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:39:51.564452+0000", 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000446272, 2024-09-13T14:19:03.860 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.861 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.862 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.863 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:03.864 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "64'1" 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.865 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549682+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549682+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550059+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.866 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.867 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.5", 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.868 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000", 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.869 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.870 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.871 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.872 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.873 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.564972+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.564972+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565253+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.874 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.875 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.4", 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000", 2024-09-13T14:19:03.876 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.877 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.878 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.879 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.880 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.374161+0000", 2024-09-13T14:19:03.881 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.374161+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.399510+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.882 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.883 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.0", 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 18, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:24:25.293852+0000", 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000494481, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.884 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.886 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.887 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.888 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.889 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.546934+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.546934+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.547223+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.890 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.891 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.6", 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000", 2024-09-13T14:19:03.892 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.893 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.894 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.895 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.896 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.897 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570753+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570753+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571004+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.898 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.899 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.7", 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.900 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000", 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.901 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.902 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.903 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.904 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.905 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605337+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605337+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.399066+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.906 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.907 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.1", 2024-09-13T14:19:03.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:12:20.334474+0000", 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000585351, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.909 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.910 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.911 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.912 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.913 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545067+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545067+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545342+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.914 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.915 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.7", 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 26, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000", 2024-09-13T14:19:03.916 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.917 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.918 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:03.919 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.920 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.921 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963216+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963216+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963528+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.922 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.923 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.6", 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:46:29.430917+0000", 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.924 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.925 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.926 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.927 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.928 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549735+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549735+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550293+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.929 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.930 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.8", 2024-09-13T14:19:03.931 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000", 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.932 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.933 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.934 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.935 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.936 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555684+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555684+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.556457+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.937 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.938 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.9", 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.939 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000", 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.940 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.941 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.942 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.943 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.944 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549291+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549291+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549793+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.945 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.946 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.9", 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.947 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000", 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.950 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.951 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.952 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.553125+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.553125+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.554955+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.953 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.954 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.8", 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.955 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000", 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.957 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.958 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.959 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549508+0000", 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549508+0000", 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549798+0000", 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.960 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:03.961 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.a", 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.962 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000", 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.964 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.965 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.966 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.967 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571523+0000", 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571523+0000", 2024-09-13T14:19:03.968 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571783+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.969 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.970 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.b", 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000", 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.971 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.972 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.973 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.974 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.975 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549022+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549022+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549324+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.976 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.977 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.b", 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.978 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000", 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.979 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.980 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.981 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.982 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.983 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.549328+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.549328+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.549648+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.984 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.985 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.a", 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000", 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.986 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.987 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.988 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.989 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.990 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572830+0000", 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572830+0000", 2024-09-13T14:19:03.991 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573588+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:03.992 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:03.993 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1d", 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000", 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:03.994 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:03.995 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:03.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:03.997 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:03.998 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549754+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549754+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550180+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:03.999 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:04.000 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.001 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1c", 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000", 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.002 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.003 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:04.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:04.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.006 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963081+0000", 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963081+0000", 2024-09-13T14:19:04.007 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963486+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.008 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.009 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1c", 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:54:51.607549+0000", 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.010 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.011 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.013 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.014 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549179+0000", 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549179+0000", 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550265+0000", 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.015 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.016 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1d", 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.017 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000", 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.018 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.021 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:04.022 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.575235+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.575235+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.575732+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.023 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.024 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1f", 2024-09-13T14:19:04.025 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000", 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.026 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.029 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:04.030 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542604+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542604+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542901+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.032 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1e", 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.033 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000", 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.037 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.038 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.554633+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.554633+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555625+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.040 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.041 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1e", 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000", 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.045 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.046 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.605752+0000", 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.605752+0000", 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.609525+0000", 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 4, 2024-09-13T14:19:04.048 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 4, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1f", 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.049 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 44, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:01:29.004573+0000", 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 570, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 570, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:04.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 4, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.053 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'4" 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_per_pool": [ 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 24, 2024-09-13T14:19:04.054 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 3, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 8, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 3, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 4, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 19, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.055 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:04.056 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 2, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 2 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 12288, 2024-09-13T14:19:04.058 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 57, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 24 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.059 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 192, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 64, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 3, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.061 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.062 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.063 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.064 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 192 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 96, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 45, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 32, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 45, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 2, 2024-09-13T14:19:04.065 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 3410, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 570, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 66, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 22, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 22, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 14, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.068 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 51, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 19 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 98304, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 10230, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:04.069 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 96 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 3, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 79, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 1, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 3, 2024-09-13T14:19:04.070 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 79, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 1, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 590368, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 590368, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.071 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 126, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 231, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 113, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1628 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 1781760, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 1771104, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:04.074 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 3 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 105, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": 4, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_by_class": { 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "ssd": { 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_bytes": 2443300864, 2024-09-13T14:19:04.075 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_ratio": 0.003181545762345195 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516984320, 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443300864 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:19:04.076 INFO:tasks.workunit.client.0.smithi026.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-13T14:19:04.077 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry show-device 2024-09-13T14:19:04.077 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:04.078 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:19:04.078 INFO:tasks.workunit.client.0.smithi026.stdout: "1e627828-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.078 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPED1D480GA_1e632c28-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.078 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141258": { 2024-09-13T14:19:04.078 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "E2010325", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e627828-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236777 2024-09-13T14:19:04.079 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:19:04.080 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:19:04.081 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:19:04.082 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 8322763 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.084 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.085 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:04.086 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 15 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "0.000000" 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.087 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1010736 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 0, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 0, 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.088 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.089 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:04.089 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.089 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:04.089 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 0, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 3253, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 114944730, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 545440628, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 1693655669, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 13000783575, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:04.092 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 0, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 20275, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 25721, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 31, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 6932 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 20275, 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:04.093 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 25721 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:04.094 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:04.095 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: E2010325", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-13T14:19:04.096 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.097 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 31 Celsius", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 0%", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 0%", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 114,944,730 [58.8 TB]", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 545,440,628 [279 TB]", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 1,693,655,669", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 13,000,783,575", 2024-09-13T14:19:04.098 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 3,253", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 20,275", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 25,721", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 6,932", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:04.099 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 31 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.100 INFO:tasks.workunit.client.0.smithi026.stdout: "1e646412-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPEDMD400G4_1e64e450-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141301": { 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "8DV101H0", 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e646412-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.101 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236781 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.102 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:19:04.103 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.104 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 27995823 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.105 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 49507808 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:04.106 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.107 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:04.108 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "63.999023" 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:04.109 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 89, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 3385, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 3399, 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 3370 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.110 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 10, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 26, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 364889334, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 1834734314, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 6215216103, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 33084752516, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:04.111 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 11, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 56859, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 55867, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 25, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 1468 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 56859, 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:04.112 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 55867 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:04.113 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: 8DV101H0", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:04.114 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:04.115 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 1 - 512 8 2", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 2 - 512 16 2", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 3 - 4096 0 0", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 4 - 4096 8 0", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 5 - 4096 64 0", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: " 6 - 4096 128 0", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:04.116 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 25 Celsius", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 10%", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 11%", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 364,889,334 [186 TB]", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 1,834,734,314 [939 TB]", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 6,215,216,103", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 33,084,752,516", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 26", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 56,859", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 55,867", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 1,468", 2024-09-13T14:19:04.117 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.118 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 25 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:19:04.119 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph telemetry show-all 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.120 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.122 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1692518364' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout:{ 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "device_report": { 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "1e627828-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPED1D480GA_1e632c28-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141258": { 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.143 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "E2010325", 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e627828-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236777 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:19:04.144 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.145 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.146 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:04.147 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 8322763 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.148 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.149 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.150 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 15 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:04.151 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "0.000000" 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1010736 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.152 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 0, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 0, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 0, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 3253, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:04.153 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 114944730, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 545440628, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 1693655669, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 13000783575, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 0, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 20275, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 25721, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 31, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 6932 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.154 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 20275, 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 25721 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.155 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:04.156 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: E2010325", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:04.157 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:12:57 2024 UTC", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.158 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 31 Celsius", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 0%", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 0%", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 114,944,730 [58.8 TB]", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 545,440,628 [279 TB]", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 1,693,655,669", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 13,000,783,575", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 3,253", 2024-09-13T14:19:04.159 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 20,275", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 25,721", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 6,932", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:04.160 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 31 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 937703088, 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 480103981056 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.161 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "1e646412-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "INTEL_SSDPEDMD400G4_1e64e450-71db-11ef-a516-ac1f6ba5cac0": { 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "20240913-141301": { 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "device": { 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "info_name": "/dev/nvme0n1", 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "/dev/nvme0n1", 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "protocol": "NVMe", 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "nvme" 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "firmware_version": "8DV101H0", 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "host_id": "1e646412-71db-11ef-a516-ac1f6ba5cac0", 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: "json_format_version": [ 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.162 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "local_time": { 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "asctime": "Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "time_t": 1726236781 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "logical_block_size": 512, 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_controller_id": 0, 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_namespaces": [ 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.163 INFO:tasks.workunit.client.0.smithi026.stdout: "capacity": { 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "formatted_lba_size": 512, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 1, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "size": { 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "utilization": { 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.164 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_number_of_namespaces": 1, 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_pci_vendor": { 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "id": 32902, 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "subsystem_id": 32902 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_add_log": { 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "Device stats": { 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_fw_downgrades": { 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.165 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "crc_error_count": { 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "end_to_end_error_detection_count": { 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "erase_fail_count": { 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.166 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "host_bytes_written": { 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 27995823 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "host_ctx_wear_used": { 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 1 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "media_bytes_read": { 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "nand_bytes_written": { 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 49507808 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_stat_indicator": { 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "pll_lock_loss_count": { 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "program_fail_count": { 2024-09-13T14:19:04.168 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "re_alloc_sectr_cnt": { 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "retry_buffer_overflow_count": { 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "soft_ecc_err_rate": { 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.169 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "thermal_throttle_status": { 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "cnt": 0, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "pct": 0 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_host_reads": { 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_media_wear": { 2024-09-13T14:19:04.170 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": "63.999023" 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "timed_workload_timer": { 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 100, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 65535 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "unexp_power_loss": { 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 0, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": 0 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "wear_leveling": { 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "normalized": 89, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "raw": { 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "avg": 3385, 2024-09-13T14:19:04.171 INFO:tasks.workunit.client.0.smithi026.stdout: "max": 3399, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 3370 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel Smart log": "nvme0n1" 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_smart_health_information_log": { 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare": 100, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "available_spare_threshold": 10, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "controller_busy_time": 26, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "critical_warning": 0, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_read": 364889334, 2024-09-13T14:19:04.172 INFO:tasks.workunit.client.0.smithi026.stdout: "data_units_written": 1834734314, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "host_reads": 6215216103, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "host_writes": 33084752516, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "media_errors": 0, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_err_log_entries": 0, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "percentage_used": 11, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycles": 56859, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_hours": 55867, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": 25, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "unsafe_shutdowns": 1468 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_vendor": "intel", 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme_version": { 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "string": "<1.2", 2024-09-13T14:19:04.173 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "power_cycle_count": 56859, 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "power_on_time": { 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "hours": 55867 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "serial_number": "deleted", 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "smart_status": { 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "nvme": { 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "passed": true 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl": { 2024-09-13T14:19:04.174 INFO:tasks.workunit.client.0.smithi026.stdout: "argv": [ 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "-x", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "--json=o", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "/dev/nvme0n1" 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "build_info": "(local build)", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "exit_status": 0, 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "output": [ 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.175 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Serial Number: deleted", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Version: 8DV101H0", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller ID: 0", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "NVMe Version: <1.2", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Number of Namespaces: 1", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Local Time is: Fri Sep 13 14:13:01 2024 UTC", 2024-09-13T14:19:04.176 INFO:tasks.workunit.client.0.smithi026.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported Power States", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-13T14:19:04.177 INFO:tasks.workunit.client.0.smithi026.stdout: " 0 + 512 0 2", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 1 - 512 8 2", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 2 - 512 16 2", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 3 - 4096 0 0", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 4 - 4096 8 0", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 5 - 4096 64 0", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: " 6 - 4096 128 0", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "Critical Warning: 0x00", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "Temperature: 25 Celsius", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare: 100%", 2024-09-13T14:19:04.178 INFO:tasks.workunit.client.0.smithi026.stdout: "Available Spare Threshold: 10%", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Percentage Used: 11%", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Read: 364,889,334 [186 TB]", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Data Units Written: 1,834,734,314 [939 TB]", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Read Commands: 6,215,216,103", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Host Write Commands: 33,084,752,516", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Controller Busy Time: 26", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Power Cycles: 56,859", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Power On Hours: 55,867", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Unsafe Shutdowns: 1,468", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Media and Data Integrity Errors: 0", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information Log Entries: 0", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "", 2024-09-13T14:19:04.179 INFO:tasks.workunit.client.0.smithi026.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "No Errors Logged", 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "" 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "svn_revision": "5155", 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "version": [ 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "temperature": { 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "current": 25 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.180 INFO:tasks.workunit.client.0.smithi026.stdout: "user_capacity": { 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "blocks": 781422768, 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 400088457216 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "report": { 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer": { 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "active": true, 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: "mode": "" 2024-09-13T14:19:04.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "channels": [ 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "channels_available": [ 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "basic", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "ident", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "crash", 2024-09-13T14:19:04.182 INFO:tasks.workunit.client.0.smithi026.stdout: "device", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "perf" 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_available": [ 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:04.183 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "collections_opted_in": [ 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_base", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "device_base", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "crash_base", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "ident_base", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_perf", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_mds_metadata", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_usage", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_usage_by_class", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_rook_v01", 2024-09-13T14:19:04.184 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_memory_metrics", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "basic_pool_options_bluestore" 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "config": { 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "active_changed": [ 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "client_check_pool_perm", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_gid", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "client_mount_uid", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "client_quota", 2024-09-13T14:19:04.185 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "daemonize", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "keyring", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_block_size", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_cache_size", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_compression", 2024-09-13T14:19:04.186 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_log", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "leveldb_write_buffer_size", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_journald", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_stderr", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_host", 2024-09-13T14:19:04.187 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:04.188 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "no_config_file", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:04.189 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_lim", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_res", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:04.190 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addr", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "public_addrv", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_default_features", 2024-09-13T14:19:04.191 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd_qos_exclude_ops", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "setgroup", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "setuser" 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_changed": [ 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_debug", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_mon_ticket_ttl", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "auth_service_ticket_ttl", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "bdev_debug_aio", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "container_image", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_asserts_on_shutdown", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mgr", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_mon", 2024-09-13T14:19:04.192 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_ms", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_osd", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "debug_paxos", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_file", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_journald", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "log_to_stderr", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_join_fs", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/allow_ptrace", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/container_init", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/migration_current", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/cephadm/use_repo_digest", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-13T14:19:04.193 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/dashboard/ssl_server_port", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/orchestrator/orchestrator", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_ident", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/channel_perf", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/description", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/enabled", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr/telemetry/organization", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_allow_pool_delete", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_clock_drift_allowed", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_cluster_log_to_file", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_data_avail_warn", 2024-09-13T14:19:04.194 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_election_default_strategy", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_max_pg_per_osd", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_mgr_mkfs_grace", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_prime_pg_temp", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_osd_reporter_subtree_level", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_pg_warn_max_object_skew", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-13T14:19:04.195 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "mon_warn_on_too_few_osds", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_bug", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "ms_die_on_old_message", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_default_list", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_class_load_list", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_misdirected_ops", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_op_order", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_pg_log_writeout", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_shutdown", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_cached_snaps", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_missing_on_start", 2024-09-13T14:19:04.196 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_debug_verify_stray_on_activate", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mclock_profile", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_memory_target_autotune", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_op_queue_cut_off", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_erasure_code_profile", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recover_clone_overlap", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_recovery_max_chunk", 2024-09-13T14:19:04.197 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_load_threshold", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_scrub_max_interval", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_shutdown_pgref_assert", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "public_network", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_cache_enabled", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_ops_log", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "rgw_enable_usage_log" 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "contact": null, 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "crashes": [], 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "created": "2024-09-13T14:09:50.810274Z", 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "crush": { 2024-09-13T14:19:04.198 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_algs": { 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "straw2": 3 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_sizes": { 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "2": 1, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "4": 2 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket_types": { 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "1": 2, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "11": 1 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "compat_weight_set": false, 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: "device_classes": [ 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: 8 2024-09-13T14:19:04.199 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_buckets": 6, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_devices": 8, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_rules": 1, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_types": 12, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_weight_sets": 0, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "tunables": { 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "allowed_bucket_algs": 54, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_fallback_tries": 0, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_local_tries": 0, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "choose_total_tries": 50, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_descend_once": 1, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_stable": 1, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "chooseleaf_vary_r": 1, 2024-09-13T14:19:04.200 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v2_rules": 0, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v3_rules": 0, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v4_buckets": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "has_v5_rules": 0, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "legacy_tunables": 0, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "minimum_required_version": "jewel", 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "optimal_tunables": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "profile": "jewel", 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables2": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables3": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "require_feature_tunables5": 1, 2024-09-13T14:19:04.201 INFO:tasks.workunit.client.0.smithi026.stdout: "straw_calc_version": 1 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "description": "upgrade test cluster", 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "fs": { 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "feature_flags": { 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "enable_multiple": true, 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_enabled_multiple": true 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "filesystems": [ 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "approx_ctime": "2024-09", 2024-09-13T14:19:04.202 INFO:tasks.workunit.client.0.smithi026.stdout: "balancer_enabled": false, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_caps": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_dns": 10, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_inos": 13, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "cached_subtrees": 2, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "ever_allowed_features": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "explicitly_allowed_features": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "files": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "max_mds": 1, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "num_data_pools": 1, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "num_in": 1, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "num_mds": 1, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": 0, 2024-09-13T14:19:04.203 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_replay": 0, 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "num_up": 1, 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "snaps": 0, 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "standby_count_wanted": 0 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "num_standby_mds": 1, 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "total_num_mds": 2 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_stats": { 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:04.204 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 20594688, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 442904, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 589824, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 433, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 1127840, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 20, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1298944, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 15103560, 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 21184512 2024-09-13T14:19:04.205 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 20135936, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 318632, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 753664, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 0, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 351, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 891784, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 12, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 853504, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 14696912, 2024-09-13T14:19:04.206 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 20135936 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 37453824, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1161672, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 188416, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 9011200, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1204, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 6991000, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 22, 2024-09-13T14:19:04.207 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2600192, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 23891104, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 46465024 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 35340288, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1132192, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 10067968, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 992, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 6107384, 2024-09-13T14:19:04.208 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 21, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2035456, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 23443816, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 45408256 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 45056000, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 1580472, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2621440, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 188416, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 3399680, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1760, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 9428008, 2024-09-13T14:19:04.209 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 22, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 3337728, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 27899936, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 48455680 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 274120704, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 952520, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 868352, 2024-09-13T14:19:04.210 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1403, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5283104, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2617600, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262514968, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 274989056 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 272736256, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 718984, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 925696, 2024-09-13T14:19:04.211 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1351, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5110576, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 46, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2580224, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261573960, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273661952 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 307822592, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 879608, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 4849664, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 720896, 2024-09-13T14:19:04.212 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1248, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4953112, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1949184, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 295191024, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 308543488 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271810560, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 756432, 2024-09-13T14:19:04.213 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 802816, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1263, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4510120, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1922304, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261869192, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 272613376 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271409152, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 755312, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.214 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 90112, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1241, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5008248, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 46, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1662976, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261230104, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 271499264 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 273170432, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 746360, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 24576, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 770048, 2024-09-13T14:19:04.215 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1284, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 5000304, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2284032, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262362648, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273940480 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 271810560, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 779064, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 16384, 2024-09-13T14:19:04.216 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 802816, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1259, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4985888, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 45, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 1976576, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 261300136, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 272613376 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "actual_memory_used": 272850944, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "central_cache_freelist": 934992, 2024-09-13T14:19:04.217 INFO:tasks.workunit.client.0.smithi026.stdout: "malloc_metadata": 2752512, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "page_heap_freelist": 0, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "released_to_os": 876544, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "spans_in_use": 1320, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "tcmalloc_page_size": 8192, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_cache_freelists": 4652552, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "thread_heaps_in_use": 47, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "transfer_cache_freelist": 2427136, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "use_by_application": 262083752, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "virtual_address_space_used": 273727488 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "hosts": { 2024-09-13T14:19:04.218 INFO:tasks.workunit.client.0.smithi026.stdout: "num": 2, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mds": 2, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mgr": 2, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_mon": 2, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_with_osd": 2 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "io_rate": { 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_stats_delta": { 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 0, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 0, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "stamp_delta": "12.002235", 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.219 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.220 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 398, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 458, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 88, 2024-09-13T14:19:04.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.223 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 0 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "leaderboard": false, 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "license": "sharing-1-0", 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "mempool": { 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.224 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.225 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.226 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.227 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.228 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.229 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 222471, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 72 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 50488, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 208 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.230 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8136, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 139 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.231 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.232 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.233 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.234 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.235 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.236 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.237 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 150088, 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 33 2024-09-13T14:19:04.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 856, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 2 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.239 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.240 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.241 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.242 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.243 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.244 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.245 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.246 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.247 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1293334, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 407 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.248 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.249 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5312, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 103 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.250 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.251 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.252 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.253 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.254 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.255 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.256 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1084470, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 356 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.257 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.258 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5312, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 103 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.259 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.260 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.261 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.262 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.263 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.264 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1143286, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 351 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 352, 2024-09-13T14:19:04.265 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.266 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6784, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 138 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12744, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1546 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.267 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10624, 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 206 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.269 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20384, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 672, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 6 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.270 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8736, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20384, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 182 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5680, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 71 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 38022, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.271 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 54150, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9794 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 133960, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10972, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 708 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.272 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 31244, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 409193, 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 90 2024-09-13T14:19:04.273 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2449096, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 197 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5896, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 67 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.274 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 509696, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 44 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 75712, 2024-09-13T14:19:04.275 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 148 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.276 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.277 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.278 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 896, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8112, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:04.279 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4640, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 58 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19182, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48126, 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8781 2024-09-13T14:19:04.280 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 124440, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9132, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 533 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.281 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27224, 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 736, 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 410401, 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 90 2024-09-13T14:19:04.282 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12096, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2388365, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4928, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 56 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.283 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 440192, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 38 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8528, 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 22 2024-09-13T14:19:04.284 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236944, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3552 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.285 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.286 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.287 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 112, 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8112, 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.288 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 18928, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4160, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 52 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4096, 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:04.289 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 46328, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8543 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 121040, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 178 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8332, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 525 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.290 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 25944, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 178 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 321941, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 68 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.291 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2405328, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 176 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4224, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 48 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.292 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 359104, 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 31 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.293 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10400, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 28 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236688, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3540 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.294 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:04.295 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.296 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 448, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 4 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8304, 2024-09-13T14:19:04.297 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3840, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 48 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 15086, 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 47276, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8775 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 124440, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9132, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.299 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 26552, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 183 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2208, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 298663, 2024-09-13T14:19:04.300 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 63 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30637, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2444010, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4928, 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 56 2024-09-13T14:19:04.301 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 428608, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 37 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.302 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10400, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 20 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.303 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:04.304 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.305 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 672, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 6 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.306 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8304, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19376, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 173 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3520, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 44 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4096, 2024-09-13T14:19:04.307 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48382, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8923 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 126480, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9324, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.308 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27320, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 186 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 736, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 1 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.309 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 913585, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 69 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 12096, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2397418, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 169 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4840, 2024-09-13T14:19:04.310 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 55 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 451776, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 39 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.311 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8944, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 18 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236432, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3528 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.312 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.313 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.314 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19152, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 224, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 2 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8208, 2024-09-13T14:19:04.315 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 19152, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 3040, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 38 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.316 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 49086, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8963 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 127160, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 9540, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 542 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.317 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 27912, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 187 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2208, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 838740, 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 51 2024-09-13T14:19:04.318 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30637, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2417468, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 181 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 5456, 2024-09-13T14:19:04.319 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 62 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 474944, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 41 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.320 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 13520, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 26 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236944, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3552 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.321 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.322 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.323 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20160, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.324 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8640, 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20160, 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 180 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2720, 2024-09-13T14:19:04.325 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 34 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 48697, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9204 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 125800, 2024-09-13T14:19:04.326 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 185 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 10564, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 699 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 28428, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 185 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.327 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 497356, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 47 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.328 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2408644, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 171 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 4576, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 52 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 451776, 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 39 2024-09-13T14:19:04.329 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 75712, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 156 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:04.330 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.331 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "bloom_filter": { 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6304, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 115 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.332 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_reader": { 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 16896, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 8 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs_file_writer": { 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 576, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Blob": { 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20048, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Buffer": { 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 1232, 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 11 2024-09-13T14:19:04.333 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_Extent": { 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 8592, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_SharedBlob": { 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 20048, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 179 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_alloc": { 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2160, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 27 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.334 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_data": { 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 293272, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 12 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_meta": { 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 51658, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 9593 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_onode": { 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 131240, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 193 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_cache_other": { 2024-09-13T14:19:04.335 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 11328, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 704 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_fsck": { 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_inline_bl": { 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 30644, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 193 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_txc": { 2024-09-13T14:19:04.336 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing": { 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 178783, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 35 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore_writing_deferred": { 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_anon": { 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 2420701, 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 226 2024-09-13T14:19:04.337 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "buffer_meta": { 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 6248, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 71 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_co": { 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 532864, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 46 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.338 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_mapbl": { 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_pglog": { 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 77480, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 193 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap": { 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 236176, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 3516 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.339 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap_mapping": { 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "pgmap": { 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_1": { 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "unittest_2": { 2024-09-13T14:19:04.340 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes": 0, 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "items": 0 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "metadata": { 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 2 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:04.341 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 2 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 2 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.342 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 2 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 3 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:19:04.343 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 3 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 3 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:04.344 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 3 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "arch": { 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "x86_64": 8 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_version": { 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.345 INFO:tasks.workunit.client.0.smithi026.stdout: "cpu": { 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "distro": { 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "centos": 8 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "distro_description": { 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "CentOS Stream 9": 8 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_description": { 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "kernel_version": { 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-13T14:19:04.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "os": { 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "Linux": 8 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_objectstore": { 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": 8 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "rotational": { 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "0": 8 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.347 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "features": { 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "optional": [], 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "persistent": [ 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "kraken", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "luminous", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "mimic", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "osdmap-prune", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "nautilus", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "octopus", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "pacific", 2024-09-13T14:19:04.348 INFO:tasks.workunit.client.0.smithi026.stdout: "elector-pinging", 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "quincy" 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv4_addr_mons": 6, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "ipv6_addr_mons": 0, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "min_mon_release": 17, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "v1_addr_mons": 3, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "v2_addr_mons": 3 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "organization": "ceph-qa", 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.349 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster_network": false, 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "require_min_compat_client": "luminous", 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "require_osd_release": "quincy" 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_perf_histograms": [ 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:04.350 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:04.351 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:04.352 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.353 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:04.354 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:04.355 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:04.356 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.357 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.358 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:04.359 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.360 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.361 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.362 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.363 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.364 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.365 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.366 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.367 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.368 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.369 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.370 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.371 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.372 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.373 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.374 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.375 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.376 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.377 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.378 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.379 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.380 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:04.381 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:04.382 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.383 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:04.384 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:04.385 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:04.386 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.387 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.388 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.389 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.390 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.391 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.392 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.393 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.394 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.395 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.396 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.397 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.398 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.399 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.400 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.401 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.402 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.403 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.404 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.405 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.406 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.407 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.408 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.409 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:04.410 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:04.411 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:04.412 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:04.413 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:04.414 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:04.415 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.416 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.417 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.418 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.419 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.420 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.421 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.422 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.423 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.424 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.425 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.426 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.427 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.428 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.429 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.430 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.431 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.432 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.433 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.434 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.435 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.436 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.437 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.438 INFO:tasks.workunit.client.0.smithi026.stdout: "axes": [ 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Latency (usec)", 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 100000, 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 99999]", 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "[100000, 199999]", 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "[200000, 399999]", 2024-09-13T14:19:04.439 INFO:tasks.workunit.client.0.smithi026.stdout: "[400000, 799999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[800000, 1599999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[1600000, 3199999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[3200000, 6399999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[6400000, 12799999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[12800000, 25599999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[25600000, 51199999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[51200000, 102399999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[102400000, 204799999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[204800000, 409599999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[409600000, 819199999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[819200000, 1638399999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[1638400000, 3276799999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[3276800000, 6553599999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[6553600000, 13107199999]", 2024-09-13T14:19:04.440 INFO:tasks.workunit.client.0.smithi026.stdout: "[13107200000, 26214399999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[26214400000, 52428799999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[52428800000, 104857599999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[104857600000, 209715199999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[209715200000, 419430399999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[419430400000, 838860799999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[838860800000, 1677721599999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[1677721600000, 3355443199999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[3355443200000, 6710886399999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[6710886400000, 13421772799999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[13421772800000, 26843545599999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[26843545600000, 53687091199999]", 2024-09-13T14:19:04.441 INFO:tasks.workunit.client.0.smithi026.stdout: "[53687091200000, None]" 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "buckets": 32, 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "min": 0, 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "name": "Request size (bytes)", 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "quant_size": 512, 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "ranges": [ 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "[None, -1]", 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 511]", 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "[512, 1023]", 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "[1024, 2047]", 2024-09-13T14:19:04.442 INFO:tasks.workunit.client.0.smithi026.stdout: "[2048, 4095]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[4096, 8191]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[8192, 16383]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[16384, 32767]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[32768, 65535]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[65536, 131071]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[131072, 262143]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[262144, 524287]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[524288, 1048575]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[1048576, 2097151]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[2097152, 4194303]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[4194304, 8388607]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[8388608, 16777215]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[16777216, 33554431]", 2024-09-13T14:19:04.443 INFO:tasks.workunit.client.0.smithi026.stdout: "[33554432, 67108863]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[67108864, 134217727]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[134217728, 268435455]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[268435456, 536870911]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[536870912, 1073741823]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[1073741824, 2147483647]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[2147483648, 4294967295]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[4294967296, 8589934591]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[8589934592, 17179869183]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[17179869184, 34359738367]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[34359738368, 68719476735]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[68719476736, 137438953471]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[137438953472, 274877906943]", 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: "[274877906944, None]" 2024-09-13T14:19:04.444 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "scale_type": "log2" 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "osds": [ 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 0, 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.445 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.446 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.447 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 1, 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 5, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 16, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.448 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.449 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 2, 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.450 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.451 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.452 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 3, 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.453 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.454 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.455 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 4, 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.456 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.457 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.458 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 5, 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.459 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.460 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.461 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 6, 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.462 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.463 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.464 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "osd_id": 7, 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "values": [ 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.465 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.466 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.467 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: ] 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "perf_counters": { 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "mds.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "mds": { 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "caps": { 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.468 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_flush_ack": { 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_grant": { 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_revoke": { 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "ceph_cap_op_trunc": { 2024-09-13T14:19:04.469 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_commit": { 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_fetch": { 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_merge": { 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "dir_split": { 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.470 INFO:tasks.workunit.client.0.smithi026.stdout: "exported_inodes": { 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "forward": { 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_cap_release": { 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_caps": { 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_caps_dirty": { 2024-09-13T14:19:04.471 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_inode_file_caps": { 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "imported_inodes": { 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes": { 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_expired": { 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.472 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_pinned": { 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "inodes_with_caps": { 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "load_cent": { 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "openino_dir_fetch": { 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "process_request_cap_release": { 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.473 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "reply_latency": { 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 80402 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "request": { 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rbytes": { 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rfiles": { 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "root_rsnaps": { 2024-09-13T14:19:04.474 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_reply": { 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "subtrees": { 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_cache": { 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_enqueue_scrub": { 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.475 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_exportdir": { 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_flush": { 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_fragmentdir": { 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_fragstats": { 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.476 INFO:tasks.workunit.client.0.smithi026.stdout: "ireq_inodestats": { 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_enqueued": { 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_prioritized": { 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_recovering_processing": { 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays": { 2024-09-13T14:19:04.477 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays_delayed": { 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_strays_enqueuing": { 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_completed": { 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_started": { 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.478 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_created": { 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_enqueued": { 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_migrated": { 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "strays_reintegrated": { 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.479 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_log": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "ev": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "evadd": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "evex": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "evexd": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "evexg": { 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.480 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "evtrm": { 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "jlat": { 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 11659984 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "replayed": { 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "seg": { 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "segadd": { 2024-09-13T14:19:04.481 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "segex": { 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "segexd": { 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "segexg": { 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "segtrm": { 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.482 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_mem": { 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "cap": { 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "cap+": { 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "cap-": { 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "dir": { 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.483 INFO:tasks.workunit.client.0.smithi026.stdout: "dir+": { 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "dir-": { 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "dn": { 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "dn+": { 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "dn-": { 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.484 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "heap": { 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 207068 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "ino": { 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "ino+": { 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "ino-": { 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "rss": { 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39072 2024-09-13T14:19:04.485 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_server": { 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "cap_acquisition_throttle": { 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "cap_revoke_eviction": { 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_request": { 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_client_session": { 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.486 INFO:tasks.workunit.client.0.smithi026.stdout: "handle_peer_request": { 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "req_create_latency": { 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getattr_latency": { 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 80402 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getfilelock_latency": { 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.487 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "req_getvxattr_latency": { 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "req_link_latency": { 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookup_latency": { 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookuphash_latency": { 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.488 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupino_latency": { 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupname_latency": { 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupparent_latency": { 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lookupsnap_latency": { 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "req_lssnap_latency": { 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mkdir_latency": { 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mknod_latency": { 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.490 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "req_mksnap_latency": { 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "req_open_latency": { 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "req_readdir_latency": { 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rename_latency": { 2024-09-13T14:19:04.491 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "req_renamesnap_latency": { 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmdir_latency": { 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmsnap_latency": { 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.492 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "req_rmxattr_latency": { 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setattr_latency": { 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setdirlayout_latency": { 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.493 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setfilelock_latency": { 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setlayout_latency": { 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "req_setxattr_latency": { 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.494 INFO:tasks.workunit.client.0.smithi026.stdout: "req_symlink_latency": { 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "req_unlink_latency": { 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "mds_sessions": { 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "average_load": { 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.495 INFO:tasks.workunit.client.0.smithi026.stdout: "avg_session_uptime": { 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 89 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "mdthresh_evicted": { 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "session_count": { 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "session_remove": { 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.496 INFO:tasks.workunit.client.0.smithi026.stdout: "sessions_open": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "sessions_stale": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "total_load": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.497 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.498 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.498 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.498 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.498 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.499 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.499 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "purge_queue": { 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executed": { 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing": { 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_high_water": { 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_ops": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_executing_ops_high_water": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "pq_item_in_journal": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.017959b3ae2599c720706d30d5d8d763c91ab264": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.501 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:04.502 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.503 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 846 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 846, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4103025 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 846, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 130997014 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.504 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 146 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.505 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 999977 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 847 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.506 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.507 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 847 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 847, 2024-09-13T14:19:04.508 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1936955855 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.509 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 847 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 847, 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8047145 2024-09-13T14:19:04.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 847, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5806 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 847, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 383991261 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.511 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 43843584 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 34340864 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9502720 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.512 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021340 2024-09-13T14:19:04.513 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271552 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.515 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70834340 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.516 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021340 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41237 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.517 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.518 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71064655 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.519 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184680893 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.520 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71171651 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.522 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26290536 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.523 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36572, 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 83811666 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.524 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:04.525 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1842, 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 611751552 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.396fb4c86f9dd7f0a0d6d62b6b76c402f26ca4fd": { 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:04.526 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 96 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:04.527 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 837 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 837, 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4077355 2024-09-13T14:19:04.528 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 837, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 81998786 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:04.529 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:04.530 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 839 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.531 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.532 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 839 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 839, 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1901971822 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:04.533 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.534 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 838 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 838, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7997714 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 838, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5735 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:04.535 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 838, 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 116998269 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42786816 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32718848 2024-09-13T14:19:04.536 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10067968 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.537 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021755 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270445 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.538 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70835032 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.540 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277021755 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41237 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.541 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.543 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71064240 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.544 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184681170 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.545 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.546 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71171374 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.547 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 30482974 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 35718, 2024-09-13T14:19:04.548 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 90740454 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.549 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1796, 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 222149850 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.550 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "mon.5d74eb5ca058ab6f2fa05025dc3922667cd3a77d": { 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "election_call": { 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "election_lose": { 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "election_win": { 2024-09-13T14:19:04.551 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_elections": { 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_sessions": { 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "session_add": { 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "session_rm": { 2024-09-13T14:19:04.552 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 191 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "session_trim": { 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "paxos": { 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "accept_timeout": { 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "begin": { 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 970 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_bytes": { 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.553 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5151277 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_keys": { 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2910 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "begin_latency": { 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 165997545 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "collect": { 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_bytes": { 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.554 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_keys": { 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_latency": { 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_timeout": { 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "collect_uncommitted": { 2024-09-13T14:19:04.555 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "commit": { 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 970 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_bytes": { 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5081495 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_keys": { 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5636 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "commit_latency": { 2024-09-13T14:19:04.556 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 970, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 140788772 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_ack_timeout": { 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "lease_timeout": { 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn": { 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "new_pn_latency": { 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.557 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 999985 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh": { 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 972 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "refresh_latency": { 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 972, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2342965308 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "restart": { 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state": { 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.558 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_bytes": { 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1627 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "share_state_keys": { 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "start_leader": { 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "start_peon": { 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.559 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state": { 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_bytes": { 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_keys": { 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "store_state_latency": { 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache": { 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1020054731 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45834240 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41205760 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2147483648 2024-09-13T14:19:04.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4628480 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:full": { 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.562 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277022402 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271122 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.563 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 70833708 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:inc": { 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 348127232 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.565 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 277022402 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38834 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.566 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71065996 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "prioritycache:kv": { 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 322961408 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184681601 2024-09-13T14:19:04.569 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.570 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.571 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71170943 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb": { 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "compact": { 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_completed": { 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lasted": { 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31667556 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.572 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_len": { 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_queue_merge": { 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_running": { 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "get_latency": { 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 43166, 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 173060035 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_delay_time": { 2024-09-13T14:19:04.573 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_memtable_time": { 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_write_wal_time": { 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.574 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_latency": { 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "submit_sync_latency": { 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2108, 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 286090462 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.0": { 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.576 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2555904 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.577 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1724416 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1585152 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.578 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.579 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.580 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.581 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.583 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4214784 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.584 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1298432 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 239524 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.585 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 35705 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.586 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 370, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 74872413 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 362, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10132492 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 370, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 437389 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 370, 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 75309802 2024-09-13T14:19:04.587 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32588 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7934 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.588 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10163 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780 2024-09-13T14:19:04.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 229 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 114, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1416171 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 272, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 238478 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 158, 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 793004 2024-09-13T14:19:04.590 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55136 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6348293 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 99, 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7152809 2024-09-13T14:19:04.591 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 99, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 557122282294 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 99, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 150024659481 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 76015175626 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.592 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55458 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 427485 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 147156442 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 184302 2024-09-13T14:19:04.593 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 87503254 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 21018746 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 901055 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 59, 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 267573 2024-09-13T14:19:04.594 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 262547009 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 25153734 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 469, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4313823 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.595 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 272236544 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271360000 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 876544 2024-09-13T14:19:04.596 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85553054 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.597 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.598 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24576 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.599 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73805922 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641647910 2024-09-13T14:19:04.600 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.601 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.602 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71381642 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.603 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57035369 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.605 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68793287 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.606 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641647910 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.607 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 278461 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.608 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71105309 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.609 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6 2024-09-13T14:19:04.610 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 604 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46338348 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.611 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3838826 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26641510 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.612 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.613 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.614 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.615 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 604 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46338348 2024-09-13T14:19:04.616 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3838826 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26641510 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.617 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 570 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305786880 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.618 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.1": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.619 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2064384 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.620 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1400832 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.621 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1261568 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.622 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.623 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.624 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3399680 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.627 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 692224 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 234021 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.628 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 24, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60822 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.629 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55445273 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 286, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9453820 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 431815 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.630 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 291, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 55877088 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31093 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14142 2024-09-13T14:19:04.631 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13337 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.632 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 683 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 234 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 116, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1837745 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 285479 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.633 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 155, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1178382 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 110282 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6930128 2024-09-13T14:19:04.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 95, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7180933 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 94, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 380165209420 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 95, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 120202842339 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.635 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 365, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 86309478108 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 365, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 47205 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 391418 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.636 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 97503526 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 128312 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 59983516 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23209152 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.637 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 309520 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 54, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 309689 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 188077449 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 27536659 2024-09-13T14:19:04.638 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 366, 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4367024 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270909440 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.639 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269975552 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 933888 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.640 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85555323 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.642 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73815941 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.643 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641664929 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.644 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.645 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71364623 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.646 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57036882 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.647 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.649 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791774 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.650 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641664929 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.651 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 252990 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.652 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71113761 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.653 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 97 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.654 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1640827 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 411266444 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 602 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.655 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 34402695 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 97, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 121526526 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 18 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2687287 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.656 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 602 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2243773 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 18, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1806843 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.657 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10415687 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.658 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4490655 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9714590 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67 2024-09-13T14:19:04.659 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1640827 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 398163470 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 27668267 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.660 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 67, 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 110005093 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.661 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305184768 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.2": { 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.662 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.663 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1716224 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.664 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1159168 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1019904 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.665 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.666 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.667 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.668 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.669 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.670 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2809856 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 692224 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.671 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 220296 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.672 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9595 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50371502 2024-09-13T14:19:04.673 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 227, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9769353 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 351683 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 232, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50723185 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.674 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42896 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 942 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.675 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7732 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 500 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 215 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.676 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 861316 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 214, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 262146 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 123, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 420533 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.677 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 101916 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8444398 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8036179 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.678 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 347940565486 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 86, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 109038186035 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 76361559862 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32640 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.679 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 583496 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 104043309 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 114195 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 75985560 2024-09-13T14:19:04.680 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15925543 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 310541 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 48, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 233233 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.681 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 205042362 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 21108256 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 293, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2204795 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.682 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 303693824 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 302964736 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 729088 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.683 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85556299 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.684 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.685 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73823157 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.686 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641672243 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.687 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.688 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71357309 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.689 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57037532 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.690 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.691 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791124 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.692 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641672243 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.693 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.694 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 247723 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71111714 2024-09-13T14:19:04.695 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.696 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.697 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 91874897 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5476516 2024-09-13T14:19:04.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42709457 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.699 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.700 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.701 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.702 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.702 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.702 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.702 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.702 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.703 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.703 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.703 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 10 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 91874897 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.704 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5476516 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 10, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42709457 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.705 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305180672 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.3": { 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.706 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.707 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1122304 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.708 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 851968 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.709 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 712704 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.710 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.711 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.712 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.714 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1908736 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.715 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 708608 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.716 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7317 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.717 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40513826 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 150, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6393392 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 239914 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.718 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40753740 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45696 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1024 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.719 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9061 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.720 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 343 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 215 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 820702 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 160, 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 248326 2024-09-13T14:19:04.721 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 69, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 222833 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.722 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3323354 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3738964 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 47, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 291757300607 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 75206048061 2024-09-13T14:19:04.723 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 234, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 577313 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 234, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 26764 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 280409 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.724 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 138019317 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68458 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71471481 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13213499 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.725 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312699 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 189006 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 226335230 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16663999 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.726 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 237, 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1908882 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1862400347 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269860864 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269017088 2024-09-13T14:19:04.727 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3138478489 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 843776 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85555622 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.729 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8192 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.730 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73819738 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.731 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641667169 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71362383 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.734 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 57037081 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.735 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.736 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68791575 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.737 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641667169 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.738 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6964 2024-09-13T14:19:04.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 247939 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.740 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71109608 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.741 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 37 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.742 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.743 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1148904 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.744 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.745 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.746 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.747 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.748 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1148904 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.749 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305197056 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.750 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.4": { 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.751 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1323008 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.752 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.753 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1011712 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 872448 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.754 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.755 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.756 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.757 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.758 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2269184 2024-09-13T14:19:04.759 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 708608 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 302921 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.760 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5538 2024-09-13T14:19:04.761 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44558699 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5843060 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.762 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 225243 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 196, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 44783942 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29553 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.763 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1061 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.764 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9990 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 376 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 231 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 93, 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 941978 2024-09-13T14:19:04.765 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 157, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195067 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 64, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 321136 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.766 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 108690 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3783863 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 42, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2954084 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 212467068807 2024-09-13T14:19:04.767 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 42, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68124457548 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 273, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8605932315 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 273, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 31816 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.768 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 330120 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 81533165 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 87630 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.769 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 52756757 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13780348 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 310686 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 40, 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 196471 2024-09-13T14:19:04.770 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 152222169 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16713138 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2274789 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.771 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268746752 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 268599296 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:04.772 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 147456 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.773 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498177 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.774 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.775 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73881279 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.776 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236333 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.777 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.778 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71793219 2024-09-13T14:19:04.779 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998785 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.780 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.781 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829871 2024-09-13T14:19:04.782 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236333 2024-09-13T14:19:04.783 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.784 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 253659 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71541688 2024-09-13T14:19:04.785 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.786 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.787 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 22 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4942962 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.788 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1022875 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4845855 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.789 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.790 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.791 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.792 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 22 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4942962 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.793 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1022875 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4845855 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.794 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305201152 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.5": { 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.795 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.796 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1167360 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.797 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 868352 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 729088 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.798 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.799 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.800 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.801 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.802 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1970176 2024-09-13T14:19:04.803 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 700416 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 6, 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 248067 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.804 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 37035 2024-09-13T14:19:04.805 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60565413 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 160, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5433846 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:04.806 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 174038 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 161, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 60739451 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 33837 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.807 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5874 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.808 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4482 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 354 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 238 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 91, 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1728477 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.809 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 147, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 273499 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 56, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1352842 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 4, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 78993 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.810 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3510043 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2918961 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 38, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 276986572867 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 41, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65698492633 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.811 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 8286324816 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 271, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 28134 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 261894 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 119335641 2024-09-13T14:19:04.812 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 82830 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 78836259 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32156842 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.813 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312074 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 33, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 176420 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 234138535 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.814 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 34407909 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 274, 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2866983 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271187968 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.815 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270417920 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 770048 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.816 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498351 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.817 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.818 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73885201 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.819 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641237637 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.820 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.821 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71791915 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.822 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998901 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.823 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.824 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829755 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.825 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641237637 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.826 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6963 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 253470 2024-09-13T14:19:04.827 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71533610 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.828 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.829 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 41 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.830 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.831 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.832 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.833 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.834 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.835 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.836 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305188864 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.837 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.6": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.838 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1458176 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.839 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.840 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1134592 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 995328 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.841 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.842 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.843 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.844 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.845 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.846 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2527232 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1290240 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.847 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.848 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 8, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19931 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 226, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 59141430 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 216, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 6136437 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.849 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 226, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 176495 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 226, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 59317925 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 1, 2024-09-13T14:19:04.850 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 33119 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4656 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.851 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 9111 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 401 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 240 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 89, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1170337 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.852 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 141, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 183675 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 52, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 943629 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.853 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2280897 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2510008 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 437209091041 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 36, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 53719950822 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.854 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 508024 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 32962 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 207075 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 99349302 2024-09-13T14:19:04.855 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 99600 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 54282641 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13158448 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.856 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 902669 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 29, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 135244 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 169326242 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15581112 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.857 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 284, 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2742347 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269860864 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 269058048 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.858 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 802816 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.859 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85498208 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.860 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.861 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73885344 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.862 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236564 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.863 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.864 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71792988 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.865 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56998805 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.866 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.867 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68829851 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.868 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641236564 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.869 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7103 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.870 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 255713 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71532300 2024-09-13T14:19:04.871 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.872 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.873 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2804 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42228473 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.874 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 15, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7772952 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29658765 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.875 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.876 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.877 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.878 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2804 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42228473 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.879 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 15, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 7772952 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 13, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 29658765 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:04.880 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305778688 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "osd.7": { 2024-09-13T14:19:04.881 INFO:tasks.workunit.client.0.smithi026.stdout: "bluefs": { 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_fallback": { 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_slow_size_fallback": { 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_db": { 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 65536 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_main": { 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.882 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit_wal": { 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_slow": { 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_sst": { 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 12288 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_written_wal": { 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1400832 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lat": { 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.883 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "compact_lock_lat": { 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "db_total_bytes": { 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "db_used_bytes": { 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 304480256 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "log_bytes": { 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1089536 2024-09-13T14:19:04.884 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "logged_bytes": { 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 950272 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_db": { 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 312868864 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_slow": { 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "max_bytes_wal": { 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.885 INFO:tasks.workunit.client.0.smithi026.stdout: "num_files": { 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 14 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "read_bytes": { 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 195697 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "read_count": { 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 23 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes": { 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1269760 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_db": { 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1273856 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.886 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_slow": { 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_bytes_wal": { 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "read_disk_count": { 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 15 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_bytes": { 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4789 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "read_prefetch_count": { 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4 2024-09-13T14:19:04.887 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_bytes": { 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_buffer_count": { 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_bytes": { 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5589 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_count": { 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 24 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes": { 2024-09-13T14:19:04.888 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_db": { 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_slow": { 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_bytes_wal": { 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "read_random_disk_count": { 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_total_bytes": { 2024-09-13T14:19:04.889 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "slow_used_bytes": { 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_total_bytes": { 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "wal_used_bytes": { 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "write_bytes": { 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2424832 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.890 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore": { 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "alloc_unit": { 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4096 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": { 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1286144 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "clist_lat": { 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 216725 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_lat": { 2024-09-13T14:19:04.891 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed": { 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_allocated": { 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "compressed_original": { 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "csum_lat": { 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 12, 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 38501 2024-09-13T14:19:04.892 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "decompress_lat": { 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_commit_lat": { 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 215, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 54544883 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_final_lat": { 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 205, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4987174 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_flush_lat": { 2024-09-13T14:19:04.893 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 215, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 124477 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "kv_sync_lat": { 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 215, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 54669360 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_keys_lat": { 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_get_values_lat": { 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 30608 2024-09-13T14:19:04.894 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_lower_bound_lat": { 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1102 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_next_lat": { 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_seek_to_first_lat": { 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 0, 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.895 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_upper_bound_lat": { 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 2, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 4540 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_hits": { 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 857 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "onode_misses": { 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 280 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "read_lat": { 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 141, 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3549987 2024-09-13T14:19:04.896 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "read_onode_meta_lat": { 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 235, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 316338 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "read_wait_aio_lat": { 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 94, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2674703 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "reads_with_retries": { 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "remove_lat": { 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 3, 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 61887 2024-09-13T14:19:04.897 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "state_aio_wait_lat": { 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1944343 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_aio_wait_lat": { 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 23, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1715724 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_cleanup_lat": { 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 23, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 276511388824 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "state_deferred_queued_lat": { 2024-09-13T14:19:04.898 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 23, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 39433865849 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "state_done_lat": { 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 490241 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "state_finishing_lat": { 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 36478 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "state_io_done_lat": { 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 201247 2024-09-13T14:19:04.899 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_commiting_lat": { 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 130867756 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_done_lat": { 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 101987 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "state_kv_queued_lat": { 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71137182 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "state_prepare_lat": { 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.900 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13519809 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": { 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 900361 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "truncate_lat": { 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 22, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 99790 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_commit_lat": { 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 217712724 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_submit_lat": { 2024-09-13T14:19:04.901 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16088295 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "txc_throttle_lat": { 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 322, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2293617 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache": { 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_bytes": { 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 1861443168 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "heap_bytes": { 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270974976 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.902 INFO:tasks.workunit.client.0.smithi026.stdout: "mapped_bytes": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 270057472 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "target_bytes": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3137352396 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "unmapped_bytes": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 917504 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:data": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 159383552 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.903 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 85488887 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.904 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 143360 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.905 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 73751305 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv": { 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 780140544 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.906 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641166657 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.907 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2128 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.908 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71862895 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:kv_onode": { 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 192937984 2024-09-13T14:19:04.909 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 67108864 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 464 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 56992591 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.910 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.911 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.912 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.912 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.912 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 68836065 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "bluestore-pricache:meta": { 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "committed_bytes": { 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 713031680 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "pri0_bytes": { 2024-09-13T14:19:04.914 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "pri10_bytes": { 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "pri11_bytes": { 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 641166657 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "pri1_bytes": { 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "pri2_bytes": { 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.915 INFO:tasks.workunit.client.0.smithi026.stdout: "pri3_bytes": { 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pri4_bytes": { 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pri5_bytes": { 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pri6_bytes": { 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 271958 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "pri7_bytes": { 2024-09-13T14:19:04.916 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "pri8_bytes": { 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "pri9_bytes": { 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "reserved_bytes": { 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 71593065 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "objecter": { 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "op_active": { 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.917 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rmw": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "osd": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 46 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "numpg_removing": { 2024-09-13T14:19:04.918 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "op": { 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 185 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_degraded": { 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "op_delayed_unreadable": { 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "op_in_bytes": { 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "op_latency": { 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 185, 2024-09-13T14:19:04.919 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 110313555 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "op_out_bytes": { 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 266926 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "op_prepare_latency": { 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 187, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 40937361 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "op_process_latency": { 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 185, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 78792637 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r": { 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 130 2024-09-13T14:19:04.920 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency": { 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 130, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19285739 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_out_bytes": { 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 266926 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_prepare_latency": { 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 130, 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 16844748 2024-09-13T14:19:04.921 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "op_r_process_latency": { 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 130, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 13572797 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw": { 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 50 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_in_bytes": { 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency": { 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 48881891 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-13T14:19:04.922 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_out_bytes": { 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_prepare_latency": { 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 21074781 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "op_rw_process_latency": { 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 50, 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 45484784 2024-09-13T14:19:04.923 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w": { 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 5 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_in_bytes": { 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency": { 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 42145925 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_prepare_latency": { 2024-09-13T14:19:04.924 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 7, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 3017832 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "op_w_process_latency": { 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "count": 5, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 19735056 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "op_wip": { 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 0 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_bytes": { 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 590368 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "recovery_ops": { 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 2 2024-09-13T14:19:04.925 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes": { 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 95995035648 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_bytes_used": { 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "value": 305774592 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": [ 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.926 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 1, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 1, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 1, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115966730, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 1781760, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 1781760, 2024-09-13T14:19:04.927 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 524, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 705536, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 590368, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 590368, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 1771104, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 201, 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 1667072 2024-09-13T14:19:04.928 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 32, 2024-09-13T14:19:04.929 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 32, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 2, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115966730, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 98304, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 98304, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 22, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:04.930 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 3410, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 3410, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 10230, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 51, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 19456 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.931 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 64, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 64, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 3, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:04.932 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115966730, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 0.0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 0, 2024-09-13T14:19:04.933 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 0 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "cache_mode": "none", 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "erasure_code_profile": {}, 2024-09-13T14:19:04.934 INFO:tasks.workunit.client.0.smithi026.stdout: "min_size": 2, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "options": {}, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_autoscale_mode": "off", 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 8, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "pgp_num": 8, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "pool": 4, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "size": 3, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "stats": { 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_raw": 727115966730, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "bytes_used": 12288, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_bytes_used": 0, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "compress_under_bytes": 0, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "data_bytes_used": 12288, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty": 0, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "max_avail": 242371985408, 2024-09-13T14:19:04.935 INFO:tasks.workunit.client.0.smithi026.stdout: "objects": 2, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_bytes_used": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "percent_used": 1.689964257423071e-08, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_bytes": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "quota_objects": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "rd": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "rd_bytes": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "stored": 19, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_data": 19, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_omap": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "stored_raw": 57, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "wr": 2, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "wr_bytes": 2048 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_bytes": 0, 2024-09-13T14:19:04.936 INFO:tasks.workunit.client.0.smithi026.stdout: "target_max_objects": 0, 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "type": "replicated" 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": { 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "mirroring_by_pool": [ 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: false 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "num_images_by_pool": [ 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pools": 1 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.937 INFO:tasks.workunit.client.0.smithi026.stdout: "report_id": "bb35b373-4dfb-4d9f-9cf9-b49d389b3568", 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "report_timestamp": "2024-09-13T14:19:03.548775", 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "report_version": 1, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "rocksdb_stats": { 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "6.15.5" 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "rook": { 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "cluster": { 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "external-mode": null, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "mon": { 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "allow-multiple-per-node": null, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "count": null, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "max-id": null, 2024-09-13T14:19:04.938 INFO:tasks.workunit.client.0.smithi026.stdout: "pvc": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "stretch": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "enabled": null 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "network": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "provider": null 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "storage": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "device-set": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "non-portable": null, 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "portable": null, 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:04.939 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "csi": { 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes": { 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "node": { 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "kubernetes-total": null, 2024-09-13T14:19:04.940 INFO:tasks.workunit.client.0.smithi026.stdout: "with-ceph-daemons": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-cephfs-plugin": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-nfs-plugin": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "with-csi-rbd-plugin": null 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "storage-class": { 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "count": { 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "bucket": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "nfs": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd": null, 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: "total": null 2024-09-13T14:19:04.941 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "version": null 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "services": {}, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_per_pg": [ 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.942 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568075+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568075+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568393+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.943 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427311+0000", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.944 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3f", 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:04.945 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:14:43.120744+0000", 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.946 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.947 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.948 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.949 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563364+0000", 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563364+0000", 2024-09-13T14:19:04.950 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.563624+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403288+0000", 2024-09-13T14:19:04.951 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3e", 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.952 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:43:38.253972+0000", 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.953 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.954 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.955 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.956 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566331+0000", 2024-09-13T14:19:04.957 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566331+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566586+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426637+0000", 2024-09-13T14:19:04.958 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3d", 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.959 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:02:03.357527+0000", 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.960 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.961 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.962 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.963 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.564496+0000", 2024-09-13T14:19:04.964 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.564496+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564769+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.965 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404976+0000", 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.966 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3c", 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:22:09.850392+0000", 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.967 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.968 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.969 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.970 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.971 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555311+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555311+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555592+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.972 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427347+0000", 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.973 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3b", 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.974 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:59:44.297275+0000", 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.975 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.976 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.977 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.978 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571310+0000", 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571310+0000", 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572204+0000", 2024-09-13T14:19:04.979 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404742+0000", 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.980 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3a", 2024-09-13T14:19:04.981 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:50:06.755716+0000", 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.982 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.983 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.984 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.985 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566403+0000", 2024-09-13T14:19:04.986 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566403+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566676+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426113+0000", 2024-09-13T14:19:04.987 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.39", 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.988 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:39:10.234619+0000", 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.989 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.990 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.991 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.992 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:04.993 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566400+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566400+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566663+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:04.994 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399106+0000", 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:04.995 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.38", 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:23:11.232404+0000", 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:04.996 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:04.997 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:04.998 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:04.999 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.000 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.569776+0000", 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.569776+0000", 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570338+0000", 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.001 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427605+0000", 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.002 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.003 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.37", 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:03:45.123915+0000", 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.004 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.005 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.006 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.007 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.008 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563959+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563959+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564796+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.009 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427114+0000", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.010 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.36", 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.011 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:16:53.815061+0000", 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.012 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.013 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.014 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.015 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.016 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566726+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566726+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567040+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.017 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426507+0000", 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.018 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.35", 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:57:11.058617+0000", 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.019 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.020 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.021 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.022 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.023 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563402+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563402+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.563679+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.024 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398688+0000", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.025 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.34", 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.026 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:19:30.129554+0000", 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.027 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.028 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.029 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.030 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.552354+0000", 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.552354+0000", 2024-09-13T14:19:05.031 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.552634+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399996+0000", 2024-09-13T14:19:05.032 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.33", 2024-09-13T14:19:05.033 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:53:46.440171+0000", 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.034 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.035 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.036 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.037 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.038 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.575737+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.575737+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.576136+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.039 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426019+0000", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.040 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.32", 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:04:52.777143+0000", 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.041 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.042 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.043 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.044 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.045 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565645+0000", 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565645+0000", 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566237+0000", 2024-09-13T14:19:05.046 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402570+0000", 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.047 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.31", 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.048 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:42:57.998325+0000", 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.049 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.050 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.051 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.052 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.053 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.964708+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.964708+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.965107+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.054 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404637+0000", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.055 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.30", 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:41:06.939390+0000", 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.056 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.057 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.058 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.059 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.060 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565091+0000", 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565091+0000", 2024-09-13T14:19:05.061 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565535+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.062 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405223+0000", 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.063 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2f", 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:06:26.216936+0000", 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.064 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.065 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.066 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.067 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.068 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.556349+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.556349+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.556686+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.069 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398039+0000", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.070 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2e", 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.071 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:23:47.030288+0000", 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.072 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.073 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.074 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.075 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.076 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.563703+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.563703+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.564825+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.077 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427100+0000", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.078 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2d", 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:17:42.374960+0000", 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.079 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.080 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.081 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.082 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.083 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568070+0000", 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568070+0000", 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568327+0000", 2024-09-13T14:19:05.084 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426232+0000", 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.085 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2c", 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.086 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:26:48.260867+0000", 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.087 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.088 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.089 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.090 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568056+0000", 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568056+0000", 2024-09-13T14:19:05.091 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568339+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.092 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426139+0000", 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2b", 2024-09-13T14:19:05.093 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:12:48.503522+0000", 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.094 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.095 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.096 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.097 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.098 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572126+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572126+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572417+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.099 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404854+0000", 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2a", 2024-09-13T14:19:05.100 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:38:45.395541+0000", 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.101 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.102 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.103 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.104 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.105 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555172+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555172+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555819+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.106 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398245+0000", 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.29", 2024-09-13T14:19:05.107 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:43:36.044605+0000", 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.108 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.109 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.110 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.111 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.112 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.559771+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.559771+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.561070+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.113 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404692+0000", 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.114 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.28", 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:09:00.907060+0000", 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.115 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.116 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.117 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.118 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.119 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.553231+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.553231+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.554841+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.120 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399852+0000", 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.121 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.27", 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:46:24.310212+0000", 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.122 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.123 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.124 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.125 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.126 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.550739+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.550739+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.551060+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.127 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427182+0000", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.128 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.26", 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:06:24.506099+0000", 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.129 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.130 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.131 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.132 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.133 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.559401+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.559401+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.559670+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.134 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402995+0000", 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.25", 2024-09-13T14:19:05.135 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:00:11.897344+0000", 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.136 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.137 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.138 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.139 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.140 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570531+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570531+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570832+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.141 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399213+0000", 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.142 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.24", 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:56:19.914646+0000", 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.143 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.144 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.145 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.146 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.147 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572783+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572783+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573053+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.148 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426207+0000", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.149 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.23", 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:52:20.579331+0000", 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.150 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.151 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.152 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.153 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.154 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568939+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568939+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.569346+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.155 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404968+0000", 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.156 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.22", 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:12:29.740749+0000", 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.157 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.158 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.159 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.160 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.161 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562267+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562267+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562543+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.162 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397771+0000", 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.163 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.21", 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:03:45.656252+0000", 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.164 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.165 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.166 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.167 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.168 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.568169+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.568169+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.568440+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.169 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425659+0000", 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.170 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.20", 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:07:12.620468+0000", 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.171 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.172 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.173 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.174 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.175 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963029+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963029+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963486+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.176 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398398+0000", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.177 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1a", 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:45:10.139607+0000", 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.178 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.179 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.180 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.181 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.182 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.539574+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.539574+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541205+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.183 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400198+0000", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.184 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1b", 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:32:51.183825+0000", 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.185 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.186 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.187 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.188 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.189 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.605583+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.605583+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.606065+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.190 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397655+0000", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.191 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1b", 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:14:21.907855+0000", 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.192 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.193 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.194 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.195 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.196 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543138+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543138+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.543444+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.197 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425880+0000", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.198 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1a", 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:31:41.794133+0000", 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.199 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.200 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.201 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.202 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.203 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.539753+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.539753+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541457+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.204 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399765+0000", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.205 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.19", 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:40:28.628583+0000", 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.206 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.207 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.208 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.209 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.210 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570884+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570884+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571137+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.211 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399167+0000", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.212 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.18", 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:31:12.814894+0000", 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.213 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.214 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.215 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.216 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.217 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545863+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545863+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546875+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.218 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404803+0000", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.219 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.18", 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:54:27.413029+0000", 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.220 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.221 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.222 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.223 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.224 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.560291+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.560291+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.560575+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.225 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402439+0000", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.226 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.19", 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:47:38.604268+0000", 2024-09-13T14:19:05.227 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.228 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.229 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.230 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.231 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545833+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545833+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546137+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.232 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427015+0000", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.233 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.17", 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.234 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:24:41.882999+0000", 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.235 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.236 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.237 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.238 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.239 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565363+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565363+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.566451+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.240 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404982+0000", 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.241 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.16", 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:12:44.150312+0000", 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.242 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.243 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.244 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.245 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.246 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.247 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540611+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540611+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.540966+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.248 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405218+0000", 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.249 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.16", 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:50:47.576826+0000", 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.250 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.251 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.252 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.253 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.254 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.573138+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.573138+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573426+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.255 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426404+0000", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.256 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.17", 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.257 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:46:35.681810+0000", 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.258 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.259 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.260 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.261 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542739+0000", 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542739+0000", 2024-09-13T14:19:05.262 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.543031+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403094+0000", 2024-09-13T14:19:05.263 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 3, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 3, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.15", 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.264 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 27, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:39:29.866737+0000", 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 90, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.265 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.266 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.267 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 4, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'3" 2024-09-13T14:19:05.268 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.269 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.550913+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.550913+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.551575+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.270 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398640+0000", 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.14", 2024-09-13T14:19:05.271 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:06:27.127421+0000", 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.272 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.273 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.274 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.275 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.276 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.538864+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.538864+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.539191+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.277 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:17:47.461329+0000", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 12, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.278 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 12, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.14", 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 36, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:34:50.274610+0000", 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.279 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 2624, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.280 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 12, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 4, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 4, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.281 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 12, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 9 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.282 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "67'12" 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.283 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.567513+0000", 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.567513+0000", 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567770+0000", 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.284 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426673+0000", 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.285 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.15", 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.286 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:20:28.895272+0000", 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.287 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.288 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.289 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.290 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.291 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545135+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545135+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545520+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.292 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426237+0000", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.293 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.13", 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 26, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:02:34.332570+0000", 2024-09-13T14:19:05.294 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 34, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.295 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.296 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.297 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 6 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.298 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.299 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.574648+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.574648+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.575023+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.300 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426065+0000", 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.301 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.12", 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T14:33:13.032176+0000", 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.302 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.303 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.304 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.305 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.306 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545255+0000", 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545255+0000", 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546988+0000", 2024-09-13T14:19:05.307 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405021+0000", 2024-09-13T14:19:05.308 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.309 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.12", 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:52:33.779661+0000", 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.310 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.311 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.312 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.313 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.314 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.315 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.316 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555391+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555391+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555682+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.317 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426955+0000", 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.318 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.13", 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:22:13.279390+0000", 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.319 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.320 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.321 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.322 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.323 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.541726+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.541726+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542050+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.324 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425985+0000", 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.325 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.11", 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.326 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:25:11.039463+0000", 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.327 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.328 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.329 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.330 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.331 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.566924+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.566924+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.567258+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.332 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425693+0000", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.333 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.10", 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:29:48.027687+0000", 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.334 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.335 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.336 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.337 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.338 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540793+0000", 2024-09-13T14:19:05.339 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540793+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.541058+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.340 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397636+0000", 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 8, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 8, 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.341 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.10", 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:16:46.458670+0000", 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.342 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.343 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 12, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 4, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 4, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 4, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.344 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 12, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.345 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'8" 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.346 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562669+0000", 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562669+0000", 2024-09-13T14:19:05.347 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562994+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.348 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426226+0000", 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.349 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.11", 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:45:51.959006+0000", 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.350 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.351 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.352 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.353 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.354 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.540386+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.540386+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.540681+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.355 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398941+0000", 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.356 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.f", 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.357 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:45:29.951889+0000", 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 22, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.358 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.359 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.360 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.361 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.362 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.558987+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.558987+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.559281+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.363 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426867+0000", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.364 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.e", 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.365 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:50:45.263027+0000", 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.366 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.367 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.368 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.369 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.370 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543782+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543782+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.544147+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.371 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398225+0000", 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.372 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.e", 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.373 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:16:12.000823+0000", 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.374 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.375 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.376 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.377 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.378 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.569357+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.569357+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.570603+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.379 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427487+0000", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.380 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.f", 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:27:48.605542+0000", 2024-09-13T14:19:05.381 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.382 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.383 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.384 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.385 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.544417+0000", 2024-09-13T14:19:05.386 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.544417+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.546363+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.387 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403205+0000", 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.388 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.d", 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:25:19.745729+0000", 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 70, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.389 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.390 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.391 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.392 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.393 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572166+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572166+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572445+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.394 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404958+0000", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.395 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.c", 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.396 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:29:13.966106+0000", 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.397 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.398 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.399 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.400 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.401 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542641+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542641+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542920+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.402 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398707+0000", 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.403 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.c", 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:01:44.067333+0000", 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.404 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.405 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.406 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.407 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.408 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571003+0000", 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571003+0000", 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.572195+0000", 2024-09-13T14:19:05.409 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426854+0000", 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.410 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.d", 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.411 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:00:42.893230+0000", 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.412 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.413 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.414 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.415 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.376845+0000", 2024-09-13T14:19:05.416 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.376845+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426886+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.417 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:33.691439+0000", 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.418 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.5", 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:07:10.386795+0000", 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000554698, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.419 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.420 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.421 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.422 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 19, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.423 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:14:57.475298+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:14:57.475298+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:14:57.479980+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 50, 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.424 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:12:56.059430+0000", 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:19:03.365452+0000", 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 123, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 49, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 123, 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.425 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "1.0", 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 213, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:06:19.586228+0000", 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 590368, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 590368, 2024-09-13T14:19:05.426 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.427 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.428 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 524, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 689, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 201, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1628 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.429 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "67'123" 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.430 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.541756+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.541756+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542042+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.431 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.404852+0000", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.432 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.3", 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.433 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:51:00.193541+0000", 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.434 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.435 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.436 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.437 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.438 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.565213+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.565213+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565510+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.439 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398634+0000", 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.440 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.2", 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:35:52.175246+0000", 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.441 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.442 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.443 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.444 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.445 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo OK 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.447 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.448 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.449 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.450 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:03 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:04 smithi145 ceph-mon[31057]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:04 smithi145 ceph-mon[31057]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:04 smithi145 ceph-mon[31057]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:04 smithi145 ceph-mon[31057]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:05.451 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:04 smithi145 ceph-mon[31057]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.452 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:22.202033+0000", 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:22.202033+0000", 2024-09-13T14:19:05.453 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426290+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.002691+0000", 2024-09-13T14:19:05.454 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.6", 2024-09-13T14:19:05.455 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:15:48.139777+0000", 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000710344, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.456 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.457 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.458 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.459 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.460 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549995+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549995+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550379+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.461 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427357+0000", 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.462 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.0", 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:32:35.292726+0000", 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.463 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.464 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.465 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.466 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.467 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.964646+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.964646+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.965011+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.468 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425608+0000", 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.469 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1", 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.470 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 31, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:36:07.136822+0000", 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.471 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.472 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.473 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.474 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.475 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605160+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605160+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403479+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.476 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.174222+0000", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.477 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.7", 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T01:55:48.959952+0000", 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000342076, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.478 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.479 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.480 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.481 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.482 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.543880+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.543880+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.544154+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.483 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398529+0000", 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.484 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1", 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:45:21.104017+0000", 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.485 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.486 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.487 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.488 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.489 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.552054+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.552054+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.553134+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.490 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402399+0000", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.491 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.0", 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.492 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:54:57.543086+0000", 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.493 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.494 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.495 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.496 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.497 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:22.202029+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:22.202029+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403217+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.498 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.173862+0000", 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.499 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.4", 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:09:48.284490+0000", 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000388303, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.500 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.501 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.502 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.503 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.504 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545610+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545610+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545901+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.505 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405168+0000", 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.506 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.2", 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:18:15.212530+0000", 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.507 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.508 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.509 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.510 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.511 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.562679+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.562679+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562994+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.512 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398541+0000", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.513 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.3", 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:54:41.112209+0000", 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.514 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.515 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.516 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.517 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.518 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605236+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605236+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.403491+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.519 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.173690+0000", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.520 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.2", 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.521 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:10:47.051775+0000", 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000975641, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 19, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.522 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.523 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 2, 2024-09-13T14:19:05.524 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 2 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "66'2" 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.525 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549305+0000", 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549305+0000", 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549591+0000", 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.526 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403019+0000", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.527 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.4", 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.528 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:32:42.257447+0000", 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.529 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.530 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.531 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.532 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.561391+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.561391+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.562976+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.533 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405183+0000", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.534 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.5", 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:29:13.350050+0000", 2024-09-13T14:19:05.535 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.536 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.537 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.538 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.539 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.604772+0000", 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.604772+0000", 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.426585+0000", 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.540 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.002319+0000", 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.541 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.3", 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.542 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:39:51.564452+0000", 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000446272, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.543 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.544 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.545 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "64'1" 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.546 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549682+0000", 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549682+0000", 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550059+0000", 2024-09-13T14:19:05.547 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427204+0000", 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.548 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.5", 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.549 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:11:57.372896+0000", 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.550 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.551 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.552 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.553 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.564972+0000", 2024-09-13T14:19:05.554 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.564972+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.565253+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402936+0000", 2024-09-13T14:19:05.555 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.4", 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.556 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:11:10.747979+0000", 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.557 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.558 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.559 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.560 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.561 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.374161+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.374161+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.399510+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.562 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:33.691085+0000", 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.563 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.0", 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 18, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:24:25.293852+0000", 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000494481, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.564 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.565 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.566 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.567 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.568 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.546934+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.546934+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.547223+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.569 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.403350+0000", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.570 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.6", 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:01:15.711601+0000", 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.571 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.572 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.573 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.574 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.575 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.570753+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.570753+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571004+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.576 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399968+0000", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.577 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.7", 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:17:06.187061+0000", 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.578 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.579 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.580 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.581 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.582 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 61, 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:16:21.605337+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:16:21.605337+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:16:25.399066+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 62, 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.583 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:16:20.359872+0000", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:34.009686+0000", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 61, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.584 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "4.1", 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 67, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 19, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:12:20.334474+0000", 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.000585351, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.585 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.586 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.587 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.588 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.589 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.545067+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.545067+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.545342+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.590 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427115+0000", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.591 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.7", 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 26, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:40:39.933697+0000", 2024-09-13T14:19:05.592 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.593 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.594 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.595 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.596 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963216+0000", 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963216+0000", 2024-09-13T14:19:05.597 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963528+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.598 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.397966+0000", 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.599 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.6", 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T00:46:29.430917+0000", 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.600 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.601 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.602 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.603 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.604 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549735+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549735+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550293+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.605 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427235+0000", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.606 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.8", 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 23, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.607 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:37:10.277832+0000", 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.608 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.609 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.610 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.611 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 4, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.555684+0000", 2024-09-13T14:19:05.612 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.555684+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.556457+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398463+0000", 2024-09-13T14:19:05.613 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.9", 2024-09-13T14:19:05.614 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:01:54.914822+0000", 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.615 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.616 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.617 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.618 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 4, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.619 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549291+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549291+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549793+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.620 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402595+0000", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.621 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.9", 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T22:05:56.960586+0000", 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.622 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.623 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.624 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.625 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: 3 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.626 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.553125+0000", 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.553125+0000", 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.554955+0000", 2024-09-13T14:19:05.627 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.399236+0000", 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.628 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.8", 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.629 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:25:21.061529+0000", 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.630 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.631 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.632 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.633 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 1, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.634 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549508+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549508+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549798+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.635 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.402834+0000", 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.636 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.a", 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T23:25:53.406832+0000", 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.637 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.638 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.639 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.640 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: 1, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: 7 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 1, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.641 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.571523+0000", 2024-09-13T14:19:05.642 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.571523+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.571783+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.643 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400073+0000", 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.644 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.b", 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T17:27:54.892023+0000", 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.645 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.646 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.647 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.648 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.649 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549022+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549022+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.549324+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.650 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.427718+0000", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.651 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.b", 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T02:00:39.224484+0000", 2024-09-13T14:19:05.652 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.653 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.654 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 1, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.655 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.656 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 6, 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.549328+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.549328+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.549648+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.657 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.425792+0000", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.658 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.a", 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T19:26:41.366247+0000", 2024-09-13T14:19:05.659 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.660 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.661 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.662 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 6, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.663 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 5, 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.572830+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.572830+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.573588+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.664 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.405098+0000", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.665 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1d", 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.666 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T15:48:01.169612+0000", 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.667 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.668 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.669 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: 6 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 5, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.670 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549754+0000", 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549754+0000", 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550180+0000", 2024-09-13T14:19:05.671 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426524+0000", 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.672 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1c", 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.673 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:52:24.871326+0000", 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.674 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.675 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:05.676 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.677 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 2, 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.963081+0000", 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.963081+0000", 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.963486+0000", 2024-09-13T14:19:05.678 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.398480+0000", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.679 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1c", 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.680 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 17, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:54:51.607549+0000", 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.681 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.682 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.683 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: 4, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: 1 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 2, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.684 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 7, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.549179+0000", 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.549179+0000", 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.550265+0000", 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.685 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426984+0000", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 2, 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.686 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 2, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1d", 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 25, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.687 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:21:22.207781+0000", 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.688 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 3, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 1, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 1, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.689 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.690 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 3, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: 7, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: 0 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 7, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'2" 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.691 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.575235+0000", 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.575235+0000", 2024-09-13T14:19:05.692 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.575732+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426376+0000", 2024-09-13T14:19:05.693 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1f", 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.694 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 21, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T21:53:39.705626+0000", 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.695 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.696 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.697 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: 5, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.698 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:22.542604+0000", 2024-09-13T14:19:05.699 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:22.542604+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:22.542901+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 52, 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.700 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400237+0000", 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 1, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 51, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 1, 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.701 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1e", 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 24, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T20:28:50.593011+0000", 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.702 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.703 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.704 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: 5 2024-09-13T14:19:05.705 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'1" 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 3, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 54, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.706 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:26.554633+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:26.554633+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:26.555625+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 55, 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.707 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:25.534586+0000", 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.400092+0000", 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 54, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.708 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "3.1e", 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 20, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T16:41:32.138558+0000", 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.709 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.710 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.711 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: 3, 2024-09-13T14:19:05.712 INFO:tasks.workunit.client.0.smithi026.stdout: 6, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: 2 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 3, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "0'0" 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": [ 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "acting_primary": 0, 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "avail_no_missing": [], 2024-09-13T14:19:05.713 INFO:tasks.workunit.client.0.smithi026.stdout: "blocked_by": [], 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "created": 51, 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "dirty_stats_invalid": false, 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_bytes_stats_invalid": false, 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "hitset_stats_invalid": false, 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_active": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_active": "2024-09-13T14:15:46.605752+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_became_peered": "2024-09-13T14:15:46.605752+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_change": "2024-09-13T14:15:46.609525+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_clean_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub": "0'0", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_deep_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_epoch_clean": 59, 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fresh": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_fullsized": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.714 INFO:tasks.workunit.client.0.smithi026.stdout: "last_peered": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub": "0'0", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_duration": 0, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_scrub_stamp": "2024-09-13T14:15:21.528831+0000", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_undegraded": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "last_unstale": "2024-09-13T14:16:25.426535+0000", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_dups_size": 0, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 4, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "log_start": "0'0", 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "manifest_stats_invalid": false, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "mapping_epoch": 58, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "object_location_counts": [], 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_scrubbed": 0, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "objects_trimmed": 0, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_stats_invalid": false, 2024-09-13T14:19:05.715 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 4, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_start": "0'0", 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "parent": "0.0", 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "parent_split_bits": 0, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "pgid": "2.1f", 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "pin_stats_invalid": false, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "purged_snaps": [], 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_epoch": 66, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "reported_seq": 44, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_duration": 0.0, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-14T18:01:29.004573+0000", 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrim_duration": 0.0, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "snaptrimq_len": 0, 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.716 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 570, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 570, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.717 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 1, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:05.718 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 4, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "state": "active+clean", 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_invalid": false, 2024-09-13T14:19:05.719 INFO:tasks.workunit.client.0.smithi026.stdout: "up": [ 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: 0, 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: 2, 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: 4 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "up_primary": 0, 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "version": "57'4" 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_per_pool": [ 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 24, 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "rbd" 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.720 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 3, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 8, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 3, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 4, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 19, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.721 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.722 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 2, 2024-09-13T14:19:05.723 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 2 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 12288, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 57, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.724 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 24 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 192, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 0, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 64, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 0, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 3, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 0, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.725 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 0, 2024-09-13T14:19:05.726 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.727 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 0 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 0, 2024-09-13T14:19:05.728 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 192 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 96, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "cephfs" 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 45, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 32, 2024-09-13T14:19:05.729 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 8, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 45, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 2, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 3410, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 570, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.730 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 66, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 22, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 22, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 14, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.731 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 51, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 19 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:05.732 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 98304, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 10230, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 96 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: { 2024-09-13T14:19:05.733 INFO:tasks.workunit.client.0.smithi026.stdout: "acting": 3, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "application": [ 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "mgr" 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "log_size": 123, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_pg": 1, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_store_stats": 3, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "ondisk_log_size": 123, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "poolid": 1, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "stat_sum": { 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes": 590368, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_hit_set_archive": 0, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_bytes_recovered": 590368, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_deep_scrub_errors": 0, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict": 0, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_kb": 0, 2024-09-13T14:19:05.734 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_full": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_evict_mode_some": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_kb": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_high": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_flush_mode_low": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_keys_recovered": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_large_omap_objects": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_legacy_snapsets": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_clones": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_object_copies": 6, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects": 2, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_degraded": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_dirty": 2, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_hit_set_archive": 0, 2024-09-13T14:19:05.735 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_manifest": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_misplaced": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_missing_on_primary": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_omap": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_pinned": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_recovered": 2, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_repaired": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_objects_unfound": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_bytes": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_omap_keys": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_promote": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read": 524, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_read_kb": 689, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_scrub_errors": 0, 2024-09-13T14:19:05.736 INFO:tasks.workunit.client.0.smithi026.stdout: "num_shallow_scrub_errors": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "num_whiteouts": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write": 201, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "num_write_kb": 1628 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "store_stats": { 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "allocated": 1781760, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "available": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_allocated": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "data_compressed_original": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "data_stored": 1771104, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "internal_metadata": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "internally_reserved": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "omap_allocated": 0, 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: "total": 0 2024-09-13T14:19:05.737 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "up": 3 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: ], 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "usage": { 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pg_num": 105, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "pools": 4, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "stats_by_class": { 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "ssd": { 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516992512, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443292672, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_bytes": 2443292672, 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_raw_ratio": 0.0031815352849662304 2024-09-13T14:19:05.738 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: }, 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: "total_avail_bytes": 765516992512, 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: "total_bytes": 767960285184, 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: "total_used_bytes": 2443292672 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout: } 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout:} 2024-09-13T14:19:05.739 INFO:tasks.workunit.client.0.smithi026.stdout:OK 2024-09-13T14:19:05.739 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:19:05.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:05.748 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:03 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:05] "GET /metrics HTTP/1.1" 200 236467 "" "Prometheus/2.43.0" 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[39182]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[39182]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[39182]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[39182]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[39182]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[27770]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[27770]: from='client.24761 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[27770]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[27770]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:05.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:04 smithi026 ceph-mon[27770]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:05.784 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-13T14:19:05.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:19:06.103 DEBUG:teuthology.parallel:result is None 2024-09-13T14:19:06.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:19:06.135 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:19:06.135 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:19:06.196 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:19:06.197 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:19:06.207 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-13T14:19:06.207 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:19:06.226 INFO:teuthology.task.print:**** done start parallel 2024-09-13T14:19:06.227 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:19:06.262 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:19:06.262 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-13T14:19:06.263 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T14:19:06.263 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-13T14:19:06.263 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-13T14:19:06.266 INFO:tasks.rados:Beginning rados... 2024-09-13T14:19:06.266 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-13T14:19:06.266 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'wip-aclamk-cbt-combined'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-aclamk-cbt-combined', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:19:06.266 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-13T14:19:06.266 INFO:tasks.rados:joining rados 2024-09-13T14:19:06.266 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:19:06.267 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:19:06.267 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-13T14:19:06.267 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T14:19:06.267 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-09-13T14:19:06.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T14:19:06.268 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-13T14:19:06.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-13T14:19:06.531 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:06.531 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:06.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:06 smithi026 ceph-mon[39182]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:06.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:06 smithi026 ceph-mon[27770]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:06 smithi145 ceph-mon[31057]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:09.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:08 smithi026 ceph-mon[39182]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:09.050 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:08 smithi026 ceph-mon[27770]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:08 smithi145 ceph-mon[31057]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 40 op/s 2024-09-13T14:19:10.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T14:19:10.639 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:11.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:10 smithi026 ceph-mon[39182]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:11.003 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:10 smithi026 ceph-mon[27770]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:10 smithi145 ceph-mon[31057]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:12.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:11 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-13T14:19:12.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:11 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-13T14:19:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:11 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-13T14:19:12.442 INFO:tasks.rados:starting run 0 out of 1 2024-09-13T14:19:12.442 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-13T14:19:12.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-13T14:19:12.919 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:13.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[39182]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:13.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-13T14:19:13.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[39182]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:19:13.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[27770]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:13.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-13T14:19:13.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:12 smithi026 ceph-mon[27770]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:19:13.064 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:12 smithi145 ceph-mon[31057]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-13T14:19:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:12 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1189646911' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-13T14:19:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:12 smithi145 ceph-mon[31057]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:19:13.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:13.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='client.14901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[39182]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='client.14901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:14 smithi026 ceph-mon[27770]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='client.14901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:19:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:14 smithi145 ceph-mon[31057]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:14.815 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:15.353 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:15] "GET /metrics HTTP/1.1" 200 236467 "" "Prometheus/2.43.0" 2024-09-13T14:19:16.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:16 smithi026 ceph-mon[39182]: pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:16 smithi026 ceph-mon[27770]: pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:16 smithi145 ceph-mon[31057]: pgmap v100: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:17.263 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:19:17.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:17 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-13T14:19:17.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:17.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:17 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-13T14:19:17.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:17.907 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:19:17.907 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (104s) 85s ago 2m 15.7M - 0.25.0 c8568f914cd2 52eac4b9b3e5 2024-09-13T14:19:17.907 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (2m) 89s ago 2m 82.6M - 9.4.7 954c08fa6188 b034aa056559 2024-09-13T14:19:17.907 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (4m) 85s ago 4m 12.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:19:17.907 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (4m) 89s ago 4m 17.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443 running (7m) 89s ago 7m 399M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7926da3ac6f2 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283,8765 running (9m) 85s ago 9m 446M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 90a812e595d2 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (9m) 85s ago 9m 47.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (7m) 89s ago 7m 40.8M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) 85s ago 7m 39.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 85s ago 2m 13.4M - 1.5.0 0da6a335fe13 0720764634c2 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (2m) 89s ago 2m 3355k - 1.5.0 0da6a335fe13 02ae27c8e5f5 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (7m) 85s ago 7m 44.3M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (6m) 85s ago 6m 43.6M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (6m) 85s ago 6m 44.2M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (6m) 85s ago 6m 41.7M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (5m) 89s ago 5m 43.9M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:19:17.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (5m) 89s ago 5m 42.2M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:19:17.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (4m) 89s ago 4m 41.5M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:19:17.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (4m) 89s ago 4m 44.0M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:19:17.909 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (99s) 89s ago 3m 29.3M - 2.43.0 a07b618ecd1d 33771513b4fc 2024-09-13T14:19:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:17 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-13T14:19:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:18.273 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:19:18.273 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:19:18.273 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:19:18.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:19:18.274 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:19:18.474 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' created 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "", 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 image", 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:19:18.599 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[39182]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[39182]: pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[39182]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[39182]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/318794084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[27770]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[27770]: pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[27770]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[27770]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:19:18.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:18 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/318794084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:18 smithi145 ceph-mon[31057]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:18 smithi145 ceph-mon[31057]: pgmap v101: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 0 B/s wr, 4 op/s 2024-09-13T14:19:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:18 smithi145 ceph-mon[31057]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:18 smithi145 ceph-mon[31057]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:19:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:18 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/318794084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:19.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-13T14:19:19.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[27770]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-13T14:19:19.493 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:19.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[39182]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-13T14:19:19.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[39182]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:19:19.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[39182]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[27770]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:19:19.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:19 smithi026 ceph-mon[27770]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:19 smithi145 ceph-mon[31057]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:19 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3521875115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-13T14:19:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:19 smithi145 ceph-mon[31057]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:19:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:19 smithi145 ceph-mon[31057]: from='client.14937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[27770]: pgmap v104: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[27770]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[27770]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:19:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[39182]: pgmap v104: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[39182]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:20 smithi026 ceph-mon[39182]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:19:20.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:20 smithi145 ceph-mon[31057]: pgmap v104: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:20.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:20 smithi145 ceph-mon[31057]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:20.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:20 smithi145 ceph-mon[31057]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:19:21.485 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-13T14:19:21.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[39182]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:19:21.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1148229871' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:21.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[27770]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:19:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1148229871' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:21 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:21 smithi145 ceph-mon[31057]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:19:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:21 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1148229871' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:21 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:19:22.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[39182]: pgmap v107: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:22.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[39182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:19:22.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[39182]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:19:22.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[27770]: pgmap v107: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:22.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[27770]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:19:22.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:22 smithi026 ceph-mon[27770]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:19:22.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:22 smithi145 ceph-mon[31057]: pgmap v107: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:22 smithi145 ceph-mon[31057]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:19:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:22 smithi145 ceph-mon[31057]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:19:23.208 DEBUG:teuthology.orchestra.run.smithi026:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight read -> 100 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight delete -> 50 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight snap_create -> 50 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight snap_remove -> 50 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight rollback -> 50 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight setattr -> 25 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight rmattr -> 25 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight copy_from -> 50 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:adding op weight append -> 100 2024-09-13T14:19:23.236 INFO:tasks.rados.rados.0.smithi026.stdout:ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout:Configuration: 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Number of operations: 4000 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Number of objects: 50 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Max in flight operations: 16 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Object size (in bytes): 4000000 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Write stride min: 400000 2024-09-13T14:19:23.237 INFO:tasks.rados.rados.0.smithi026.stdout: Write stride max: 800000 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:1: write initial oid 1 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:1: seq_num 0 ranges {0=24576} 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:1: writing smithi02683976-1 from 0 to 24576 tid 1 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:2: write initial oid 2 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:2: seq_num 1 ranges {0=24576} 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:2: writing smithi02683976-2 from 0 to 24576 tid 1 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:3: write initial oid 3 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:3: seq_num 2 ranges {0=40960} 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:3: writing smithi02683976-3 from 0 to 40960 tid 1 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:4: write initial oid 4 2024-09-13T14:19:23.491 INFO:tasks.rados.rados.0.smithi026.stdout:4: seq_num 3 ranges {0=57344} 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:4: writing smithi02683976-4 from 0 to 57344 tid 1 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:5: write initial oid 5 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:5: seq_num 4 ranges {0=73728} 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:5: writing smithi02683976-5 from 0 to 73728 tid 1 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:6: write initial oid 6 2024-09-13T14:19:23.492 INFO:tasks.rados.rados.0.smithi026.stdout:6: seq_num 5 ranges {0=90112} 2024-09-13T14:19:23.493 INFO:tasks.rados.rados.0.smithi026.stdout:6: writing smithi02683976-6 from 0 to 90112 tid 1 2024-09-13T14:19:23.493 INFO:tasks.rados.rados.0.smithi026.stdout:7: write initial oid 7 2024-09-13T14:19:23.493 INFO:tasks.rados.rados.0.smithi026.stdout:7: seq_num 6 ranges {0=106496} 2024-09-13T14:19:23.496 INFO:tasks.rados.rados.0.smithi026.stdout:7: writing smithi02683976-7 from 0 to 106496 tid 1 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:8: write initial oid 8 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:8: seq_num 7 ranges {0=122880} 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:8: writing smithi02683976-8 from 0 to 122880 tid 1 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:9: write initial oid 9 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:9: seq_num 8 ranges {0=139264} 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:9: writing smithi02683976-9 from 0 to 139264 tid 1 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:2: finishing write tid 1 to smithi02683976-2 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:1: finishing write tid 1 to smithi02683976-1 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:1: finishing write tid 2 to smithi02683976-1 2024-09-13T14:19:23.497 INFO:tasks.rados.rados.0.smithi026.stdout:3: finishing write tid 1 to smithi02683976-3 2024-09-13T14:19:23.498 INFO:tasks.rados.rados.0.smithi026.stdout:3: finishing write tid 2 to smithi02683976-3 2024-09-13T14:19:23.498 INFO:tasks.rados.rados.0.smithi026.stdout:10: write initial oid 10 2024-09-13T14:19:23.498 INFO:tasks.rados.rados.0.smithi026.stdout:10: seq_num 9 ranges {0=155648} 2024-09-13T14:19:23.498 INFO:tasks.rados.rados.0.smithi026.stdout:10: writing smithi02683976-10 from 0 to 155648 tid 1 2024-09-13T14:19:23.498 INFO:tasks.rados.rados.0.smithi026.stdout:11: write initial oid 11 2024-09-13T14:19:23.499 INFO:tasks.rados.rados.0.smithi026.stdout:4: finishing write tid 1 to smithi02683976-4 2024-09-13T14:19:23.499 INFO:tasks.rados.rados.0.smithi026.stdout:2: finishing write tid 2 to smithi02683976-2 2024-09-13T14:19:23.499 INFO:tasks.rados.rados.0.smithi026.stdout:5: finishing write tid 1 to smithi02683976-5 2024-09-13T14:19:23.499 INFO:tasks.rados.rados.0.smithi026.stdout:4: finishing write tid 2 to smithi02683976-4 2024-09-13T14:19:23.499 INFO:tasks.rados.rados.0.smithi026.stdout:11: seq_num 10 ranges {0=172032} 2024-09-13T14:19:23.500 INFO:tasks.rados.rados.0.smithi026.stdout:11: writing smithi02683976-11 from 0 to 172032 tid 1 2024-09-13T14:19:23.500 INFO:tasks.rados.rados.0.smithi026.stdout:7: finishing write tid 1 to smithi02683976-7 2024-09-13T14:19:23.500 INFO:tasks.rados.rados.0.smithi026.stdout:7: finishing write tid 2 to smithi02683976-7 2024-09-13T14:19:23.500 INFO:tasks.rados.rados.0.smithi026.stdout:12: write initial oid 12 2024-09-13T14:19:23.501 INFO:tasks.rados.rados.0.smithi026.stdout:12: seq_num 11 ranges {0=188416} 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:12: writing smithi02683976-12 from 0 to 188416 tid 1 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:2: finishing write tid 3 to smithi02683976-2 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:5: finishing write tid 2 to smithi02683976-5 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:6: finishing write tid 1 to smithi02683976-6 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:6: finishing write tid 2 to smithi02683976-6 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:3: finishing write tid 3 to smithi02683976-3 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 4 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:7: finishing write tid 3 to smithi02683976-7 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:19:23.503 INFO:tasks.rados.rados.0.smithi026.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:2: done (11 left) 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:3: done (10 left) 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:7: done (9 left) 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:13: write initial oid 13 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:13: seq_num 12 ranges {0=204800} 2024-09-13T14:19:23.504 INFO:tasks.rados.rados.0.smithi026.stdout:13: writing smithi02683976-13 from 0 to 204800 tid 1 2024-09-13T14:19:23.505 INFO:tasks.rados.rados.0.smithi026.stdout:14: write initial oid 14 2024-09-13T14:19:23.505 INFO:tasks.rados.rados.0.smithi026.stdout:14: seq_num 13 ranges {0=221184} 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:14: writing smithi02683976-14 from 0 to 221184 tid 1 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:9: finishing write tid 1 to smithi02683976-9 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:9: finishing write tid 2 to smithi02683976-9 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:6: finishing write tid 3 to smithi02683976-6 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:11: finishing write tid 1 to smithi02683976-11 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:11: finishing write tid 2 to smithi02683976-11 2024-09-13T14:19:23.507 INFO:tasks.rados.rados.0.smithi026.stdout:1: finishing write tid 3 to smithi02683976-1 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:4: finishing write tid 3 to smithi02683976-4 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:9: finishing write tid 3 to smithi02683976-9 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 4 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:5: finishing write tid 3 to smithi02683976-5 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 6 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:11: finishing write tid 3 to smithi02683976-11 2024-09-13T14:19:23.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:8: finishing write tid 1 to smithi02683976-8 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:8: finishing write tid 2 to smithi02683976-8 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:1: done (10 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:4: done (9 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:5: done (8 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:6: done (7 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:9: done (6 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:11: done (5 left) 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:15: write initial oid 15 2024-09-13T14:19:23.509 INFO:tasks.rados.rados.0.smithi026.stdout:15: seq_num 14 ranges {0=237568} 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:15: writing smithi02683976-15 from 0 to 237568 tid 1 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:10: finishing write tid 1 to smithi02683976-10 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:10: finishing write tid 2 to smithi02683976-10 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:8: finishing write tid 3 to smithi02683976-8 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 2 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:10: finishing write tid 3 to smithi02683976-10 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:8: done (5 left) 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:10: done (4 left) 2024-09-13T14:19:23.510 INFO:tasks.rados.rados.0.smithi026.stdout:16: write initial oid 16 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:16: seq_num 15 ranges {0=253952} 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:16: writing smithi02683976-16 from 0 to 253952 tid 1 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:12: finishing write tid 1 to smithi02683976-12 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:12: finishing write tid 2 to smithi02683976-12 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:14: finishing write tid 1 to smithi02683976-14 2024-09-13T14:19:23.511 INFO:tasks.rados.rados.0.smithi026.stdout:14: finishing write tid 2 to smithi02683976-14 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:12: finishing write tid 3 to smithi02683976-12 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:13: finishing write tid 1 to smithi02683976-13 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:13: finishing write tid 2 to smithi02683976-13 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:12: done (4 left) 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:17: write initial oid 17 2024-09-13T14:19:23.512 INFO:tasks.rados.rados.0.smithi026.stdout:17: seq_num 16 ranges {0=270336} 2024-09-13T14:19:23.513 INFO:tasks.rados.rados.0.smithi026.stdout:17: writing smithi02683976-17 from 0 to 270336 tid 1 2024-09-13T14:19:23.513 INFO:tasks.rados.rados.0.smithi026.stdout:18: write initial oid 18 2024-09-13T14:19:23.513 INFO:tasks.rados.rados.0.smithi026.stdout:18: seq_num 17 ranges {0=286720} 2024-09-13T14:19:23.515 INFO:tasks.rados.rados.0.smithi026.stdout:18: writing smithi02683976-18 from 0 to 286720 tid 1 2024-09-13T14:19:23.515 INFO:tasks.rados.rados.0.smithi026.stdout:19: write initial oid 19 2024-09-13T14:19:23.515 INFO:tasks.rados.rados.0.smithi026.stdout:19: seq_num 18 ranges {0=303104} 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:19: writing smithi02683976-19 from 0 to 303104 tid 1 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:14: finishing write tid 3 to smithi02683976-14 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:15: finishing write tid 1 to smithi02683976-15 2024-09-13T14:19:23.517 INFO:tasks.rados.rados.0.smithi026.stdout:15: finishing write tid 2 to smithi02683976-15 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:13: finishing write tid 3 to smithi02683976-13 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 2 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:15: finishing write tid 3 to smithi02683976-15 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:16: finishing write tid 1 to smithi02683976-16 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:16: finishing write tid 2 to smithi02683976-16 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:16: finishing write tid 3 to smithi02683976-16 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 4 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:17: finishing write tid 1 to smithi02683976-17 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:17: finishing write tid 2 to smithi02683976-17 2024-09-13T14:19:23.518 INFO:tasks.rados.rados.0.smithi026.stdout:18: finishing write tid 1 to smithi02683976-18 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:13: done (6 left) 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:14: done (5 left) 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:15: done (4 left) 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:16: done (3 left) 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:20: write initial oid 20 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:20: seq_num 19 ranges {0=319488} 2024-09-13T14:19:23.519 INFO:tasks.rados.rados.0.smithi026.stdout:20: writing smithi02683976-20 from 0 to 319488 tid 1 2024-09-13T14:19:23.520 INFO:tasks.rados.rados.0.smithi026.stdout:21: write initial oid 21 2024-09-13T14:19:23.520 INFO:tasks.rados.rados.0.smithi026.stdout:21: seq_num 20 ranges {0=344064} 2024-09-13T14:19:23.521 INFO:tasks.rados.rados.0.smithi026.stdout:21: writing smithi02683976-21 from 0 to 344064 tid 1 2024-09-13T14:19:23.521 INFO:tasks.rados.rados.0.smithi026.stdout:22: write initial oid 22 2024-09-13T14:19:23.521 INFO:tasks.rados.rados.0.smithi026.stdout:22: seq_num 21 ranges {0=360448} 2024-09-13T14:19:23.524 INFO:tasks.rados.rados.0.smithi026.stdout:22: writing smithi02683976-22 from 0 to 360448 tid 1 2024-09-13T14:19:23.524 INFO:tasks.rados.rados.0.smithi026.stdout:23: write initial oid 23 2024-09-13T14:19:23.524 INFO:tasks.rados.rados.0.smithi026.stdout:23: seq_num 22 ranges {0=376832} 2024-09-13T14:19:23.526 INFO:tasks.rados.rados.0.smithi026.stdout:23: writing smithi02683976-23 from 0 to 376832 tid 1 2024-09-13T14:19:23.526 INFO:tasks.rados.rados.0.smithi026.stdout:24: write initial oid 24 2024-09-13T14:19:23.526 INFO:tasks.rados.rados.0.smithi026.stdout:24: seq_num 23 ranges {0=393216} 2024-09-13T14:19:23.529 INFO:tasks.rados.rados.0.smithi026.stdout:24: writing smithi02683976-24 from 0 to 393216 tid 1 2024-09-13T14:19:23.529 INFO:tasks.rados.rados.0.smithi026.stdout:25: write initial oid 25 2024-09-13T14:19:23.529 INFO:tasks.rados.rados.0.smithi026.stdout:18: finishing write tid 2 to smithi02683976-18 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:17: finishing write tid 3 to smithi02683976-17 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 2 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:18: finishing write tid 3 to smithi02683976-18 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 4 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:19: finishing write tid 1 to smithi02683976-19 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:19: finishing write tid 2 to smithi02683976-19 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:19: finishing write tid 3 to smithi02683976-19 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 6 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:20: finishing write tid 1 to smithi02683976-20 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:20: finishing write tid 2 to smithi02683976-20 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:21: finishing write tid 1 to smithi02683976-21 2024-09-13T14:19:23.530 INFO:tasks.rados.rados.0.smithi026.stdout:21: finishing write tid 2 to smithi02683976-21 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:20: finishing write tid 3 to smithi02683976-20 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 8 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:22: finishing write tid 1 to smithi02683976-22 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:21: finishing write tid 3 to smithi02683976-21 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 8 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:22: finishing write tid 2 to smithi02683976-22 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:22: finishing write tid 3 to smithi02683976-22 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 4 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:23: finishing write tid 1 to smithi02683976-23 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:23: finishing write tid 2 to smithi02683976-23 2024-09-13T14:19:23.531 INFO:tasks.rados.rados.0.smithi026.stdout:25: seq_num 24 ranges {0=409600} 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:25: writing smithi02683976-25 from 0 to 409600 tid 1 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:17: done (8 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:18: done (7 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:19: done (6 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:20: done (5 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:21: done (4 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:22: done (3 left) 2024-09-13T14:19:23.532 INFO:tasks.rados.rados.0.smithi026.stdout:26: write initial oid 26 2024-09-13T14:19:23.533 INFO:tasks.rados.rados.0.smithi026.stdout:26: seq_num 25 ranges {0=425984} 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:26: writing smithi02683976-26 from 0 to 425984 tid 1 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:23: finishing write tid 3 to smithi02683976-23 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:24: finishing write tid 1 to smithi02683976-24 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:24: finishing write tid 2 to smithi02683976-24 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:24: finishing write tid 3 to smithi02683976-24 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 4 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:25: finishing write tid 1 to smithi02683976-25 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:25: finishing write tid 2 to smithi02683976-25 2024-09-13T14:19:23.535 INFO:tasks.rados.rados.0.smithi026.stdout:23: done (3 left) 2024-09-13T14:19:23.536 INFO:tasks.rados.rados.0.smithi026.stdout:24: done (2 left) 2024-09-13T14:19:23.536 INFO:tasks.rados.rados.0.smithi026.stdout:27: write initial oid 27 2024-09-13T14:19:23.536 INFO:tasks.rados.rados.0.smithi026.stdout:27: seq_num 26 ranges {0=442368} 2024-09-13T14:19:23.537 INFO:tasks.rados.rados.0.smithi026.stdout:27: writing smithi02683976-27 from 0 to 442368 tid 1 2024-09-13T14:19:23.537 INFO:tasks.rados.rados.0.smithi026.stdout:28: write initial oid 28 2024-09-13T14:19:23.537 INFO:tasks.rados.rados.0.smithi026.stdout:28: seq_num 27 ranges {0=458752} 2024-09-13T14:19:23.540 INFO:tasks.rados.rados.0.smithi026.stdout:28: writing smithi02683976-28 from 0 to 458752 tid 1 2024-09-13T14:19:23.540 INFO:tasks.rados.rados.0.smithi026.stdout:29: write initial oid 29 2024-09-13T14:19:23.540 INFO:tasks.rados.rados.0.smithi026.stdout:29: seq_num 28 ranges {0=475136} 2024-09-13T14:19:23.544 INFO:tasks.rados.rados.0.smithi026.stdout:29: writing smithi02683976-29 from 0 to 475136 tid 1 2024-09-13T14:19:23.544 INFO:tasks.rados.rados.0.smithi026.stdout:30: write initial oid 30 2024-09-13T14:19:23.544 INFO:tasks.rados.rados.0.smithi026.stdout:30: seq_num 29 ranges {0=491520} 2024-09-13T14:19:23.547 INFO:tasks.rados.rados.0.smithi026.stdout:30: writing smithi02683976-30 from 0 to 491520 tid 1 2024-09-13T14:19:23.547 INFO:tasks.rados.rados.0.smithi026.stdout:31: write initial oid 31 2024-09-13T14:19:23.547 INFO:tasks.rados.rados.0.smithi026.stdout:31: seq_num 30 ranges {0=507904} 2024-09-13T14:19:23.550 INFO:tasks.rados.rados.0.smithi026.stdout:31: writing smithi02683976-31 from 0 to 507904 tid 1 2024-09-13T14:19:23.550 INFO:tasks.rados.rados.0.smithi026.stdout:32: write initial oid 32 2024-09-13T14:19:23.550 INFO:tasks.rados.rados.0.smithi026.stdout:32: seq_num 31 ranges {0=524288} 2024-09-13T14:19:23.554 INFO:tasks.rados.rados.0.smithi026.stdout:32: writing smithi02683976-32 from 0 to 524288 tid 1 2024-09-13T14:19:23.554 INFO:tasks.rados.rados.0.smithi026.stdout:33: write initial oid 33 2024-09-13T14:19:23.554 INFO:tasks.rados.rados.0.smithi026.stdout:33: seq_num 32 ranges {0=540672} 2024-09-13T14:19:23.557 INFO:tasks.rados.rados.0.smithi026.stdout:33: writing smithi02683976-33 from 0 to 540672 tid 1 2024-09-13T14:19:23.557 INFO:tasks.rados.rados.0.smithi026.stdout:34: write initial oid 34 2024-09-13T14:19:23.557 INFO:tasks.rados.rados.0.smithi026.stdout:34: seq_num 33 ranges {0=557056} 2024-09-13T14:19:23.561 INFO:tasks.rados.rados.0.smithi026.stdout:34: writing smithi02683976-34 from 0 to 557056 tid 1 2024-09-13T14:19:23.561 INFO:tasks.rados.rados.0.smithi026.stdout:35: write initial oid 35 2024-09-13T14:19:23.562 INFO:tasks.rados.rados.0.smithi026.stdout:35: seq_num 34 ranges {0=573440} 2024-09-13T14:19:23.565 INFO:tasks.rados.rados.0.smithi026.stdout:35: writing smithi02683976-35 from 0 to 573440 tid 1 2024-09-13T14:19:23.565 INFO:tasks.rados.rados.0.smithi026.stdout:36: write initial oid 36 2024-09-13T14:19:23.565 INFO:tasks.rados.rados.0.smithi026.stdout:36: seq_num 35 ranges {0=589824} 2024-09-13T14:19:23.569 INFO:tasks.rados.rados.0.smithi026.stdout:36: writing smithi02683976-36 from 0 to 589824 tid 1 2024-09-13T14:19:23.569 INFO:tasks.rados.rados.0.smithi026.stdout:37: write initial oid 37 2024-09-13T14:19:23.569 INFO:tasks.rados.rados.0.smithi026.stdout:37: seq_num 36 ranges {0=606208} 2024-09-13T14:19:23.573 INFO:tasks.rados.rados.0.smithi026.stdout:37: writing smithi02683976-37 from 0 to 606208 tid 1 2024-09-13T14:19:23.573 INFO:tasks.rados.rados.0.smithi026.stdout:38: write initial oid 38 2024-09-13T14:19:23.573 INFO:tasks.rados.rados.0.smithi026.stdout:38: seq_num 37 ranges {0=622592} 2024-09-13T14:19:23.577 INFO:tasks.rados.rados.0.smithi026.stdout:38: writing smithi02683976-38 from 0 to 622592 tid 1 2024-09-13T14:19:23.577 INFO:tasks.rados.rados.0.smithi026.stdout:39: write initial oid 39 2024-09-13T14:19:23.577 INFO:tasks.rados.rados.0.smithi026.stdout:39: seq_num 38 ranges {0=638976} 2024-09-13T14:19:23.581 INFO:tasks.rados.rados.0.smithi026.stdout:39: writing smithi02683976-39 from 0 to 638976 tid 1 2024-09-13T14:19:23.581 INFO:tasks.rados.rados.0.smithi026.stdout:40: write initial oid 40 2024-09-13T14:19:23.581 INFO:tasks.rados.rados.0.smithi026.stdout:40: seq_num 39 ranges {0=663552} 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:40: writing smithi02683976-40 from 0 to 663552 tid 1 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:25: finishing write tid 3 to smithi02683976-25 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:25: done (15 left) 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:41: write initial oid 41 2024-09-13T14:19:23.586 INFO:tasks.rados.rados.0.smithi026.stdout:41: seq_num 40 ranges {0=679936} 2024-09-13T14:19:23.590 INFO:tasks.rados.rados.0.smithi026.stdout:41: writing smithi02683976-41 from 0 to 679936 tid 1 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:26: finishing write tid 1 to smithi02683976-26 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:26: finishing write tid 2 to smithi02683976-26 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:26: finishing write tid 3 to smithi02683976-26 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:27: finishing write tid 1 to smithi02683976-27 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:27: finishing write tid 2 to smithi02683976-27 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:27: finishing write tid 3 to smithi02683976-27 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 8 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:28: finishing write tid 1 to smithi02683976-28 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:28: finishing write tid 2 to smithi02683976-28 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:28: finishing write tid 3 to smithi02683976-28 2024-09-13T14:19:23.591 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 4 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:29: finishing write tid 1 to smithi02683976-29 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:29: finishing write tid 2 to smithi02683976-29 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:29: finishing write tid 3 to smithi02683976-29 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 10 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:26: done (15 left) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:27: done (14 left) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:28: done (13 left) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:29: done (12 left) 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:42: write initial oid 42 2024-09-13T14:19:23.592 INFO:tasks.rados.rados.0.smithi026.stdout:42: seq_num 41 ranges {0=696320} 2024-09-13T14:19:23.594 INFO:tasks.rados.rados.0.smithi026.stdout:42: writing smithi02683976-42 from 0 to 696320 tid 1 2024-09-13T14:19:23.595 INFO:tasks.rados.rados.0.smithi026.stdout:43: write initial oid 43 2024-09-13T14:19:23.595 INFO:tasks.rados.rados.0.smithi026.stdout:43: seq_num 42 ranges {0=712704} 2024-09-13T14:19:23.600 INFO:tasks.rados.rados.0.smithi026.stdout:43: writing smithi02683976-43 from 0 to 712704 tid 1 2024-09-13T14:19:23.600 INFO:tasks.rados.rados.0.smithi026.stdout:44: write initial oid 44 2024-09-13T14:19:23.600 INFO:tasks.rados.rados.0.smithi026.stdout:44: seq_num 43 ranges {0=729088} 2024-09-13T14:19:23.604 INFO:tasks.rados.rados.0.smithi026.stdout:44: writing smithi02683976-44 from 0 to 729088 tid 1 2024-09-13T14:19:23.604 INFO:tasks.rados.rados.0.smithi026.stdout:45: write initial oid 45 2024-09-13T14:19:23.604 INFO:tasks.rados.rados.0.smithi026.stdout:45: seq_num 44 ranges {0=745472} 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout:45: writing smithi02683976-45 from 0 to 745472 tid 1 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout:30: finishing write tid 1 to smithi02683976-30 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout:30: finishing write tid 2 to smithi02683976-30 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout:30: finishing write tid 3 to smithi02683976-30 2024-09-13T14:19:23.610 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 6 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:31: finishing write tid 1 to smithi02683976-31 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:31: finishing write tid 2 to smithi02683976-31 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:31: finishing write tid 3 to smithi02683976-31 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 10 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:32: finishing write tid 1 to smithi02683976-32 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:30: done (15 left) 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:31: done (14 left) 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:46: write initial oid 46 2024-09-13T14:19:23.611 INFO:tasks.rados.rados.0.smithi026.stdout:46: seq_num 45 ranges {0=761856} 2024-09-13T14:19:23.614 INFO:tasks.rados.rados.0.smithi026.stdout:46: writing smithi02683976-46 from 0 to 761856 tid 1 2024-09-13T14:19:23.614 INFO:tasks.rados.rados.0.smithi026.stdout:47: write initial oid 47 2024-09-13T14:19:23.614 INFO:tasks.rados.rados.0.smithi026.stdout:47: seq_num 46 ranges {0=778240} 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:47: writing smithi02683976-47 from 0 to 778240 tid 1 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:32: finishing write tid 2 to smithi02683976-32 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:32: finishing write tid 3 to smithi02683976-32 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 8 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:33: finishing write tid 1 to smithi02683976-33 2024-09-13T14:19:23.620 INFO:tasks.rados.rados.0.smithi026.stdout:33: finishing write tid 2 to smithi02683976-33 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:33: finishing write tid 3 to smithi02683976-33 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 10 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:34: finishing write tid 1 to smithi02683976-34 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:34: finishing write tid 2 to smithi02683976-34 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:34: finishing write tid 3 to smithi02683976-34 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 4 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:35: finishing write tid 1 to smithi02683976-35 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:35: finishing write tid 2 to smithi02683976-35 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:32: done (15 left) 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:33: done (14 left) 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:34: done (13 left) 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:48: write initial oid 48 2024-09-13T14:19:23.621 INFO:tasks.rados.rados.0.smithi026.stdout:48: seq_num 47 ranges {0=794624} 2024-09-13T14:19:23.624 INFO:tasks.rados.rados.0.smithi026.stdout:48: writing smithi02683976-48 from 0 to 794624 tid 1 2024-09-13T14:19:23.624 INFO:tasks.rados.rados.0.smithi026.stdout:49: write initial oid 49 2024-09-13T14:19:23.624 INFO:tasks.rados.rados.0.smithi026.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-13T14:19:23.627 INFO:tasks.rados.rados.0.smithi026.stdout:49: writing smithi02683976-49 from 0 to 409600 tid 1 2024-09-13T14:19:23.630 INFO:tasks.rados.rados.0.smithi026.stdout:49: writing smithi02683976-49 from 409600 to 811008 tid 2 2024-09-13T14:19:23.630 INFO:tasks.rados.rados.0.smithi026.stdout:50: write initial oid 50 2024-09-13T14:19:23.630 INFO:tasks.rados.rados.0.smithi026.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-13T14:19:23.632 INFO:tasks.rados.rados.0.smithi026.stdout:50: writing smithi02683976-50 from 0 to 425984 tid 1 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:50: writing smithi02683976-50 from 425984 to 827392 tid 2 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:35: finishing write tid 3 to smithi02683976-35 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 12 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:35: done (15 left) 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:51: copy_from oid 2 from oid 11 current snap is 0 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:36: finishing write tid 1 to smithi02683976-36 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:36: finishing write tid 2 to smithi02683976-36 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:36: finishing write tid 3 to smithi02683976-36 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 14 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-13T14:19:23.636 INFO:tasks.rados.rados.0.smithi026.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-13T14:19:23.637 INFO:tasks.rados.rados.0.smithi026.stdout:37: finishing write tid 1 to smithi02683976-37 2024-09-13T14:19:23.637 INFO:tasks.rados.rados.0.smithi026.stdout:36: done (15 left) 2024-09-13T14:19:23.637 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 0 2024-09-13T14:19:23.637 INFO:tasks.rados.rados.0.smithi026.stdout:52: seq_num 50 ranges {409600=442368,851968=401408} 2024-09-13T14:19:23.640 INFO:tasks.rados.rados.0.smithi026.stdout:52: writing smithi02683976-25 from 409600 to 851968 tid 1 2024-09-13T14:19:23.644 INFO:tasks.rados.rados.0.smithi026.stdout:52: writing smithi02683976-25 from 851968 to 1253376 tid 2 2024-09-13T14:19:23.644 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:19:23.644 INFO:tasks.rados.rados.0.smithi026.stdout:37: finishing write tid 2 to smithi02683976-37 2024-09-13T14:19:23.644 INFO:tasks.rados.rados.0.smithi026.stdout:37: finishing write tid 3 to smithi02683976-37 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 2 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:38: finishing write tid 1 to smithi02683976-38 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:38: finishing write tid 2 to smithi02683976-38 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:38: finishing write tid 3 to smithi02683976-38 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 10 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:39: finishing write tid 1 to smithi02683976-39 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:39: finishing write tid 2 to smithi02683976-39 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:39: finishing write tid 3 to smithi02683976-39 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 8 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-13T14:19:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:40: finishing write tid 1 to smithi02683976-40 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:40: finishing write tid 2 to smithi02683976-40 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:37: done (15 left) 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:38: done (14 left) 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:39: done (13 left) 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:53: rmattr oid 20 current snap is 0 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:53: done (13 left) 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:54: delete oid 10 current snap is 0 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:40: finishing write tid 3 to smithi02683976-40 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 6 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:41: finishing write tid 1 to smithi02683976-41 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:41: finishing write tid 2 to smithi02683976-41 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:41: finishing write tid 3 to smithi02683976-41 2024-09-13T14:19:23.646 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 6 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:42: finishing write tid 1 to smithi02683976-42 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:42: finishing write tid 2 to smithi02683976-42 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:42: finishing write tid 3 to smithi02683976-42 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 4 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:43: finishing write tid 1 to smithi02683976-43 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:43: finishing write tid 2 to smithi02683976-43 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:43: finishing write tid 3 to smithi02683976-43 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 12 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:44: finishing write tid 1 to smithi02683976-44 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:44: finishing write tid 2 to smithi02683976-44 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:44: finishing write tid 3 to smithi02683976-44 2024-09-13T14:19:23.647 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 2 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:45: finishing write tid 1 to smithi02683976-45 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:45: finishing write tid 2 to smithi02683976-45 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:45: finishing write tid 3 to smithi02683976-45 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 2 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:46: finishing write tid 1 to smithi02683976-46 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:46: finishing write tid 2 to smithi02683976-46 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:46: finishing write tid 3 to smithi02683976-46 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 4 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:47: finishing write tid 1 to smithi02683976-47 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:47: finishing write tid 2 to smithi02683976-47 2024-09-13T14:19:23.648 INFO:tasks.rados.rados.0.smithi026.stdout:47: finishing write tid 3 to smithi02683976-47 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:48: finishing write tid 1 to smithi02683976-48 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:48: finishing write tid 2 to smithi02683976-48 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:48: finishing write tid 3 to smithi02683976-48 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 4 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:49: finishing write tid 1 to smithi02683976-49 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:49: finishing write tid 2 to smithi02683976-49 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:49: finishing write tid 3 to smithi02683976-49 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:49: finishing write tid 4 to smithi02683976-49 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:50: finishing write tid 1 to smithi02683976-50 2024-09-13T14:19:23.649 INFO:tasks.rados.rados.0.smithi026.stdout:50: finishing write tid 2 to smithi02683976-50 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:50: finishing write tid 3 to smithi02683976-50 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:50: finishing write tid 4 to smithi02683976-50 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 13 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:51: finishing copy_from racing read to smithi02683976-2 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:51: finishing copy_from to smithi02683976-2 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-13T14:19:23.650 INFO:tasks.rados.rados.0.smithi026.stdout:52: finishing write tid 1 to smithi02683976-25 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:52: finishing write tid 2 to smithi02683976-25 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:40: done (13 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:41: done (12 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:42: done (11 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:43: done (10 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:44: done (9 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:45: done (8 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:46: done (7 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:47: done (6 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:48: done (5 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:49: done (4 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:50: done (3 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:51: done (2 left) 2024-09-13T14:19:23.652 INFO:tasks.rados.rados.0.smithi026.stdout:54: done (1 left) 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:55: read oid 15 snap -1 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:56: rollback oid 31 current snap is 0 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:56: done (2 left) 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:57: read oid 28 snap -1 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:59: rollback oid 45 current snap is 0 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:59: done (4 left) 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:60: copy_from oid 22 from oid 48 current snap is 0 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:61: delete oid 1 current snap is 0 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:52: finishing write tid 3 to smithi02683976-25 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:52: finishing write tid 4 to smithi02683976-25 2024-09-13T14:19:23.653 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 12 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:52: left oid 25 (ObjNum 50 snap 0 seq_num 50) 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:52: done (5 left) 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:61: done (4 left) 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:62: read oid 24 snap -1 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:62: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:63: snap_create 2024-09-13T14:19:23.654 INFO:tasks.rados.rados.0.smithi026.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-13T14:19:23.657 INFO:tasks.rados.rados.0.smithi026.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-13T14:19:23.672 INFO:tasks.rados.rados.0.smithi026.stdout:62: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-13T14:19:23.684 INFO:tasks.rados.rados.0.smithi026.stdout:60: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:19:23.684 INFO:tasks.rados.rados.0.smithi026.stdout:60: finishing copy_from to smithi02683976-22 2024-09-13T14:19:23.684 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 4 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-13T14:19:23.684 INFO:tasks.rados.rados.0.smithi026.stdout:58: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:19:23.685 INFO:tasks.rados.rados.0.smithi026.stdout:58: finishing copy_from to smithi02683976-29 2024-09-13T14:19:23.685 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:23 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/2715741500' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:23 smithi026 ceph-mon[27770]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:23.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:23 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/2715741500' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:23 smithi026 ceph-mon[39182]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:23 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/2715741500' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:23 smithi145 ceph-mon[31057]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-13T14:19:24.494 INFO:tasks.rados.rados.0.smithi026.stdout:55: done (5 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:57: done (4 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:58: done (3 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:60: done (2 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:62: done (1 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:63: done (0 left) 2024-09-13T14:19:24.495 INFO:tasks.rados.rados.0.smithi026.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-13T14:19:24.496 INFO:tasks.rados.rados.0.smithi026.stdout:65: read oid 30 snap -1 2024-09-13T14:19:24.496 INFO:tasks.rados.rados.0.smithi026.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-13T14:19:24.496 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 1 2024-09-13T14:19:24.496 INFO:tasks.rados.rados.0.smithi026.stdout:66: seq_num 53 ranges {376832=491520,868352=401408} 2024-09-13T14:19:24.499 INFO:tasks.rados.rados.0.smithi026.stdout:66: writing smithi02683976-23 from 376832 to 868352 tid 1 2024-09-13T14:19:24.502 INFO:tasks.rados.rados.0.smithi026.stdout:66: writing smithi02683976-23 from 868352 to 1269760 tid 2 2024-09-13T14:19:24.502 INFO:tasks.rados.rados.0.smithi026.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-13T14:19:24.517 INFO:tasks.rados.rados.0.smithi026.stdout:64: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:64: finishing copy_from to smithi02683976-15 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:64: done (2 left) 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:65: done (1 left) 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:67: rollback oid 6 current snap is 1 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 0 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: finishing write tid 1 to smithi02683976-23 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: finishing write tid 2 to smithi02683976-23 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: finishing write tid 3 to smithi02683976-23 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: finishing write tid 4 to smithi02683976-23 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 14 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-13T14:19:24.518 INFO:tasks.rados.rados.0.smithi026.stdout:66: done (1 left) 2024-09-13T14:19:24.519 INFO:tasks.rados.rados.0.smithi026.stdout:68: setattr oid 30 current snap is 1 2024-09-13T14:19:24.519 INFO:tasks.rados.rados.0.smithi026.stdout:69: copy_from oid 37 from oid 3 current snap is 1 2024-09-13T14:19:24.519 INFO:tasks.rados.rados.0.smithi026.stdout:70: read oid 15 snap 0 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:70: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:71: copy_from oid 14 from oid 41 current snap is 1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:72: rollback oid 38 current snap is 1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 0 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:73: read oid 46 snap 0 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:73: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:74: rmattr oid 18 current snap is 1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:74: done (7 left) 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:75: copy_from oid 42 from oid 12 current snap is 1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:76: read oid 26 snap -1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:76: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:77: copy_from oid 39 from oid 33 current snap is 1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:78: read oid 25 snap -1 2024-09-13T14:19:24.520 INFO:tasks.rados.rados.0.smithi026.stdout:78: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-13T14:19:24.521 INFO:tasks.rados.rados.0.smithi026.stdout:79: setattr oid 45 current snap is 1 2024-09-13T14:19:24.521 INFO:tasks.rados.rados.0.smithi026.stdout:80: snap_create 2024-09-13T14:19:24.522 INFO:tasks.rados.rados.0.smithi026.stdout:72: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:72: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:72: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 18 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:67: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:67: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:67: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:19:24.524 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 17 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-13T14:19:24.525 INFO:tasks.rados.rados.0.smithi026.stdout:70: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-13T14:19:24.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 10 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-13T14:19:24.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 4 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:24.535 INFO:tasks.rados.rados.0.smithi026.stdout:69: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:19:24.535 INFO:tasks.rados.rados.0.smithi026.stdout:73: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-13T14:19:24.557 INFO:tasks.rados.rados.0.smithi026.stdout:77: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:19:24.557 INFO:tasks.rados.rados.0.smithi026.stdout:76: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-13T14:19:24.569 INFO:tasks.rados.rados.0.smithi026.stdout:78: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:77: finishing copy_from to smithi02683976-39 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 10 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:69: finishing copy_from to smithi02683976-37 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 4 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:71: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:71: finishing copy_from to smithi02683976-14 2024-09-13T14:19:24.612 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 6 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-13T14:19:24.613 INFO:tasks.rados.rados.0.smithi026.stdout:75: finishing copy_from racing read to smithi02683976-42 2024-09-13T14:19:24.613 INFO:tasks.rados.rados.0.smithi026.stdout:75: finishing copy_from to smithi02683976-42 2024-09-13T14:19:24.613 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-13T14:19:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[39182]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[39182]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[39182]: Cluster is now healthy 2024-09-13T14:19:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[39182]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-13T14:19:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[39182]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:19:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[27770]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[27770]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[27770]: Cluster is now healthy 2024-09-13T14:19:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[27770]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-13T14:19:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:24 smithi026 ceph-mon[27770]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:19:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:24 smithi145 ceph-mon[31057]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:24 smithi145 ceph-mon[31057]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:24 smithi145 ceph-mon[31057]: Cluster is now healthy 2024-09-13T14:19:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:24 smithi145 ceph-mon[31057]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-13T14:19:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:24 smithi145 ceph-mon[31057]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:19:25.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:25 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:25] "GET /metrics HTTP/1.1" 200 236507 "" "Prometheus/2.43.0" 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:67: done (12 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:68: done (11 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:69: done (10 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:70: done (9 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:71: done (8 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:72: done (7 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:73: done (6 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:75: done (5 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:76: done (4 left) 2024-09-13T14:19:25.499 INFO:tasks.rados.rados.0.smithi026.stdout:77: done (3 left) 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:78: done (2 left) 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:79: done (1 left) 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:80: done (0 left) 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:81: rollback oid 40 current snap is 2 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 40 to 1 2024-09-13T14:19:25.500 INFO:tasks.rados.rados.0.smithi026.stdout:82: snap_create 2024-09-13T14:19:25.502 INFO:tasks.rados.rados.0.smithi026.stdout:81: finishing rollback tid 0 to smithi02683976-40 2024-09-13T14:19:25.502 INFO:tasks.rados.rados.0.smithi026.stdout:81: finishing rollback tid 1 to smithi02683976-40 2024-09-13T14:19:25.502 INFO:tasks.rados.rados.0.smithi026.stdout:81: finishing rollback tid 2 to smithi02683976-40 2024-09-13T14:19:25.503 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 14 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-13T14:19:25.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:25 smithi026 ceph-mon[39182]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:19:25.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:25 smithi026 ceph-mon[27770]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:19:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:25 smithi145 ceph-mon[31057]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:81: done (1 left) 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:82: done (0 left) 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:83: rollback oid 9 current snap is 3 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 0 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:84: read oid 20 snap 1 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:85: read oid 41 snap -1 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:86: read oid 5 snap -1 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:86: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:87: read oid 4 snap -1 2024-09-13T14:19:26.501 INFO:tasks.rados.rados.0.smithi026.stdout:87: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-13T14:19:26.502 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 3 2024-09-13T14:19:26.502 INFO:tasks.rados.rados.0.smithi026.stdout:88: seq_num 54 ranges {40960=507904,548864=401408} 2024-09-13T14:19:26.508 INFO:tasks.rados.rados.0.smithi026.stdout:88: writing smithi02683976-3 from 40960 to 548864 tid 1 2024-09-13T14:19:26.510 INFO:tasks.rados.rados.0.smithi026.stdout:88: writing smithi02683976-3 from 548864 to 950272 tid 2 2024-09-13T14:19:26.511 INFO:tasks.rados.rados.0.smithi026.stdout:89: read oid 27 snap -1 2024-09-13T14:19:26.511 INFO:tasks.rados.rados.0.smithi026.stdout:89: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-13T14:19:26.511 INFO:tasks.rados.rados.0.smithi026.stdout:87: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:83: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:83: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:87: done (6 left) 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 3 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:83: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 21 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-13T14:19:26.514 INFO:tasks.rados.rados.0.smithi026.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-13T14:19:26.535 INFO:tasks.rados.rados.0.smithi026.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-13T14:19:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:86: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-13T14:19:26.547 INFO:tasks.rados.rados.0.smithi026.stdout:88: finishing write tid 1 to smithi02683976-3 2024-09-13T14:19:26.547 INFO:tasks.rados.rados.0.smithi026.stdout:88: finishing write tid 2 to smithi02683976-3 2024-09-13T14:19:26.548 INFO:tasks.rados.rados.0.smithi026.stdout:90: seq_num 55 ranges {1253376=524288,1777664=401408} 2024-09-13T14:19:26.557 INFO:tasks.rados.rados.0.smithi026.stdout:90: writing smithi02683976-25 from 1253376 to 1777664 tid 1 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:90: writing smithi02683976-25 from 1777664 to 2179072 tid 2 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:83: done (6 left) 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:84: done (5 left) 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:85: done (4 left) 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:86: done (3 left) 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:91: rollback oid 33 current snap is 3 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 2 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:88: finishing write tid 3 to smithi02683976-3 2024-09-13T14:19:26.561 INFO:tasks.rados.rados.0.smithi026.stdout:88: finishing write tid 4 to smithi02683976-3 2024-09-13T14:19:26.562 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 19 (ObjNum 54 snap 3 seq_num 54) dirty exists 2024-09-13T14:19:26.562 INFO:tasks.rados.rados.0.smithi026.stdout:88: left oid 3 (ObjNum 54 snap 3 seq_num 54) 2024-09-13T14:19:26.562 INFO:tasks.rados.rados.0.smithi026.stdout:89: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:88: done (3 left) 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:89: done (2 left) 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:92: rollback oid 10 current snap is 3 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 0 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:93: snap_create 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:90: finishing write tid 1 to smithi02683976-25 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:90: finishing write tid 2 to smithi02683976-25 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:90: finishing write tid 3 to smithi02683976-25 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:91: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:90: finishing write tid 4 to smithi02683976-25 2024-09-13T14:19:26.579 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 18 (ObjNum 55 snap 3 seq_num 55) dirty exists 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:90: left oid 25 (ObjNum 55 snap 3 seq_num 55) 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:91: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:91: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 23 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:92: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:19:26.580 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 0 (ObjNum 33 snap 0 seq_num 519307056) dirty dne 2024-09-13T14:19:26.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:26 smithi026 ceph-mon[39182]: pgmap v112: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:26.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:26 smithi026 ceph-mon[39182]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:19:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:26 smithi026 ceph-mon[27770]: pgmap v112: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:26 smithi026 ceph-mon[27770]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:19:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:26 smithi145 ceph-mon[31057]: pgmap v112: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:26 smithi145 ceph-mon[31057]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:19:27.509 INFO:tasks.rados.rados.0.smithi026.stdout:90: done (3 left) 2024-09-13T14:19:27.509 INFO:tasks.rados.rados.0.smithi026.stdout:91: done (2 left) 2024-09-13T14:19:27.510 INFO:tasks.rados.rados.0.smithi026.stdout:92: done (1 left) 2024-09-13T14:19:27.510 INFO:tasks.rados.rados.0.smithi026.stdout:93: done (0 left) 2024-09-13T14:19:27.510 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 4 2024-09-13T14:19:27.510 INFO:tasks.rados.rados.0.smithi026.stdout:94: seq_num 56 ranges {442368=540672,983040=401408} 2024-09-13T14:19:27.515 INFO:tasks.rados.rados.0.smithi026.stdout:94: writing smithi02683976-27 from 442368 to 983040 tid 1 2024-09-13T14:19:27.518 INFO:tasks.rados.rados.0.smithi026.stdout:94: writing smithi02683976-27 from 983040 to 1384448 tid 2 2024-09-13T14:19:27.518 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 4 2024-09-13T14:19:27.518 INFO:tasks.rados.rados.0.smithi026.stdout:95: seq_num 57 ranges {540672=557056,1097728=401408} 2024-09-13T14:19:27.525 INFO:tasks.rados.rados.0.smithi026.stdout:95: writing smithi02683976-39 from 540672 to 1097728 tid 1 2024-09-13T14:19:27.527 INFO:tasks.rados.rados.0.smithi026.stdout:95: writing smithi02683976-39 from 1097728 to 1499136 tid 2 2024-09-13T14:19:27.528 INFO:tasks.rados.rados.0.smithi026.stdout:96: read oid 45 snap -1 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:96: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: finishing write tid 1 to smithi02683976-27 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: finishing write tid 2 to smithi02683976-27 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: finishing write tid 3 to smithi02683976-27 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: finishing write tid 4 to smithi02683976-27 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 27 (ObjNum 56 snap 4 seq_num 56) dirty exists 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: left oid 27 (ObjNum 56 snap 4 seq_num 56) 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:94: done (2 left) 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:97: rmattr oid 22 current snap is 4 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:97: done (2 left) 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:98: read oid 33 snap 0 2024-09-13T14:19:27.529 INFO:tasks.rados.rados.0.smithi026.stdout:98: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:27.530 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 4 2024-09-13T14:19:27.530 INFO:tasks.rados.rados.0.smithi026.stdout:99: seq_num 58 ranges {57344=573440,630784=401408} 2024-09-13T14:19:27.533 INFO:tasks.rados.rados.0.smithi026.stdout:99: writing smithi02683976-4 from 57344 to 630784 tid 1 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:99: writing smithi02683976-4 from 630784 to 1032192 tid 2 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:100: copy_from oid 16 from oid 50 current snap is 4 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:95: finishing write tid 1 to smithi02683976-39 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:95: finishing write tid 2 to smithi02683976-39 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:95: finishing write tid 3 to smithi02683976-39 2024-09-13T14:19:27.536 INFO:tasks.rados.rados.0.smithi026.stdout:98: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:27.553 INFO:tasks.rados.rados.0.smithi026.stdout:95: finishing write tid 4 to smithi02683976-39 2024-09-13T14:19:27.553 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 22 (ObjNum 57 snap 4 seq_num 57) dirty exists 2024-09-13T14:19:27.553 INFO:tasks.rados.rados.0.smithi026.stdout:95: left oid 39 (ObjNum 57 snap 4 seq_num 57) 2024-09-13T14:19:27.553 INFO:tasks.rados.rados.0.smithi026.stdout:96: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:27.576 INFO:tasks.rados.rados.0.smithi026.stdout:95: done (4 left) 2024-09-13T14:19:27.576 INFO:tasks.rados.rados.0.smithi026.stdout:96: done (3 left) 2024-09-13T14:19:27.576 INFO:tasks.rados.rados.0.smithi026.stdout:98: done (2 left) 2024-09-13T14:19:27.576 INFO:tasks.rados.rados.0.smithi026.stdout:101: snap_remove snap 1 2024-09-13T14:19:27.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:27 smithi026 ceph-mon[27770]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:19:27.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:27 smithi026 ceph-mon[39182]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:19:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:27 smithi145 ceph-mon[31057]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:101: done (2 left) 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:102: copy_from oid 17 from oid 22 current snap is 4 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:99: finishing write tid 1 to smithi02683976-4 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:99: finishing write tid 2 to smithi02683976-4 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:99: finishing write tid 3 to smithi02683976-4 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:99: finishing write tid 4 to smithi02683976-4 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 14 (ObjNum 58 snap 4 seq_num 58) dirty exists 2024-09-13T14:19:28.517 INFO:tasks.rados.rados.0.smithi026.stdout:99: left oid 4 (ObjNum 58 snap 4 seq_num 58) 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:100: finishing copy_from racing read to smithi02683976-16 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:100: finishing copy_from to smithi02683976-16 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 13 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:99: done (2 left) 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:100: done (1 left) 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:103: read oid 14 snap 3 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:103: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 4 2024-09-13T14:19:28.518 INFO:tasks.rados.rados.0.smithi026.stdout:104: seq_num 59 ranges {811008=598016,1409024=401408} 2024-09-13T14:19:28.525 INFO:tasks.rados.rados.0.smithi026.stdout:104: writing smithi02683976-49 from 811008 to 1409024 tid 1 2024-09-13T14:19:28.529 INFO:tasks.rados.rados.0.smithi026.stdout:104: writing smithi02683976-49 from 1409024 to 1810432 tid 2 2024-09-13T14:19:28.529 INFO:tasks.rados.rados.0.smithi026.stdout:105: delete oid 43 current snap is 4 2024-09-13T14:19:28.529 INFO:tasks.rados.rados.0.smithi026.stdout:103: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-13T14:19:28.549 INFO:tasks.rados.rados.0.smithi026.stdout:102: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:19:28.549 INFO:tasks.rados.rados.0.smithi026.stdout:102: finishing copy_from to smithi02683976-17 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 4 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:104: finishing write tid 1 to smithi02683976-49 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:104: finishing write tid 2 to smithi02683976-49 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:104: finishing write tid 3 to smithi02683976-49 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:104: finishing write tid 4 to smithi02683976-49 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 21 (ObjNum 59 snap 4 seq_num 59) dirty exists 2024-09-13T14:19:28.550 INFO:tasks.rados.rados.0.smithi026.stdout:104: left oid 49 (ObjNum 59 snap 4 seq_num 59) 2024-09-13T14:19:28.551 INFO:tasks.rados.rados.0.smithi026.stdout:102: done (3 left) 2024-09-13T14:19:28.551 INFO:tasks.rados.rados.0.smithi026.stdout:103: done (2 left) 2024-09-13T14:19:28.551 INFO:tasks.rados.rados.0.smithi026.stdout:104: done (1 left) 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:105: done (0 left) 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:106: read oid 43 snap -1 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:106: expect deleted 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:107: rollback oid 38 current snap is 4 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 3 2024-09-13T14:19:28.552 INFO:tasks.rados.rados.0.smithi026.stdout:108: delete oid 12 current snap is 4 2024-09-13T14:19:28.553 INFO:tasks.rados.rados.0.smithi026.stdout:106: done (2 left) 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:108: done (1 left) 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:109: read oid 9 snap 2 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:109: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:110: rollback oid 14 current snap is 4 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 3 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:111: delete oid 13 current snap is 4 2024-09-13T14:19:28.554 INFO:tasks.rados.rados.0.smithi026.stdout:107: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:19:28.555 INFO:tasks.rados.rados.0.smithi026.stdout:111: done (3 left) 2024-09-13T14:19:28.555 INFO:tasks.rados.rados.0.smithi026.stdout:112: setattr oid 45 current snap is 4 2024-09-13T14:19:28.556 INFO:tasks.rados.rados.0.smithi026.stdout:109: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-13T14:19:28.560 INFO:tasks.rados.rados.0.smithi026.stdout:107: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:19:28.560 INFO:tasks.rados.rados.0.smithi026.stdout:107: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 24 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:110: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:110: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:110: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 28 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:107: done (3 left) 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:109: done (2 left) 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:110: done (1 left) 2024-09-13T14:19:28.561 INFO:tasks.rados.rados.0.smithi026.stdout:113: snap_create 2024-09-13T14:19:28.562 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:28.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:28 smithi026 ceph-mon[27770]: pgmap v115: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:28.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:28 smithi026 ceph-mon[27770]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:19:28.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:28 smithi026 ceph-mon[39182]: pgmap v115: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:28.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:28 smithi026 ceph-mon[39182]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:19:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:28 smithi145 ceph-mon[31057]: pgmap v115: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:28 smithi145 ceph-mon[31057]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:19:29.521 INFO:tasks.rados.rados.0.smithi026.stdout:112: done (1 left) 2024-09-13T14:19:29.521 INFO:tasks.rados.rados.0.smithi026.stdout:113: done (0 left) 2024-09-13T14:19:29.521 INFO:tasks.rados.rados.0.smithi026.stdout:114: copy_from oid 25 from oid 6 current snap is 5 2024-09-13T14:19:29.521 INFO:tasks.rados.rados.0.smithi026.stdout:115: setattr oid 23 current snap is 5 2024-09-13T14:19:29.521 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 5 2024-09-13T14:19:29.522 INFO:tasks.rados.rados.0.smithi026.stdout:116: seq_num 63 ranges {172032=663552,835584=401408} 2024-09-13T14:19:29.527 INFO:tasks.rados.rados.0.smithi026.stdout:116: writing smithi02683976-15 from 172032 to 835584 tid 1 2024-09-13T14:19:29.530 INFO:tasks.rados.rados.0.smithi026.stdout:116: writing smithi02683976-15 from 835584 to 1236992 tid 2 2024-09-13T14:19:29.530 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 5 2024-09-13T14:19:29.530 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 30 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-13T14:19:29.530 INFO:tasks.rados.rados.0.smithi026.stdout:114: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:19:29.530 INFO:tasks.rados.rados.0.smithi026.stdout:117: seq_num 64 ranges {319488=679936,999424=401408} 2024-09-13T14:19:29.535 INFO:tasks.rados.rados.0.smithi026.stdout:117: writing smithi02683976-20 from 319488 to 999424 tid 1 2024-09-13T14:19:29.538 INFO:tasks.rados.rados.0.smithi026.stdout:117: writing smithi02683976-20 from 999424 to 1400832 tid 2 2024-09-13T14:19:29.538 INFO:tasks.rados.rados.0.smithi026.stdout:115: done (3 left) 2024-09-13T14:19:29.538 INFO:tasks.rados.rados.0.smithi026.stdout:118: rollback oid 14 current snap is 5 2024-09-13T14:19:29.538 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 0 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:114: finishing copy_from to smithi02683976-25 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 17 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:116: finishing write tid 1 to smithi02683976-15 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:116: finishing write tid 2 to smithi02683976-15 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:114: done (3 left) 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:119: read oid 22 snap -1 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:119: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 5 2024-09-13T14:19:29.539 INFO:tasks.rados.rados.0.smithi026.stdout:120: seq_num 65 ranges {663552=696320,1359872=401408} 2024-09-13T14:19:29.546 INFO:tasks.rados.rados.0.smithi026.stdout:120: writing smithi02683976-40 from 663552 to 1359872 tid 1 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:120: writing smithi02683976-40 from 1359872 to 1761280 tid 2 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:121: snap_create 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:116: finishing write tid 3 to smithi02683976-15 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:116: finishing write tid 4 to smithi02683976-15 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 20 (ObjNum 63 snap 5 seq_num 63) dirty exists 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:116: left oid 15 (ObjNum 63 snap 5 seq_num 63) 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:117: finishing write tid 1 to smithi02683976-20 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:117: finishing write tid 2 to smithi02683976-20 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:118: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:118: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:117: finishing write tid 3 to smithi02683976-20 2024-09-13T14:19:29.549 INFO:tasks.rados.rados.0.smithi026.stdout:118: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:19:29.550 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 34 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-13T14:19:29.550 INFO:tasks.rados.rados.0.smithi026.stdout:117: finishing write tid 4 to smithi02683976-20 2024-09-13T14:19:29.550 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 25 (ObjNum 64 snap 5 seq_num 64) dirty exists 2024-09-13T14:19:29.550 INFO:tasks.rados.rados.0.smithi026.stdout:117: left oid 20 (ObjNum 64 snap 5 seq_num 64) 2024-09-13T14:19:29.550 INFO:tasks.rados.rados.0.smithi026.stdout:119: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:120: finishing write tid 1 to smithi02683976-40 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:120: finishing write tid 2 to smithi02683976-40 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:120: finishing write tid 3 to smithi02683976-40 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:120: finishing write tid 4 to smithi02683976-40 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 25 (ObjNum 65 snap 5 seq_num 65) dirty exists 2024-09-13T14:19:29.573 INFO:tasks.rados.rados.0.smithi026.stdout:120: left oid 40 (ObjNum 65 snap 5 seq_num 65) 2024-09-13T14:19:29.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:29 smithi026 ceph-mon[27770]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:19:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:29 smithi026 ceph-mon[39182]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:19:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:29 smithi145 ceph-mon[31057]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:116: done (5 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:117: done (4 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:118: done (3 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:119: done (2 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:120: done (1 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:121: done (0 left) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:122: read oid 48 snap 3 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:123: read oid 49 snap 4 2024-09-13T14:19:30.525 INFO:tasks.rados.rados.0.smithi026.stdout:123: expect (ObjNum 59 snap 4 seq_num 59) 2024-09-13T14:19:30.526 INFO:tasks.rados.rados.0.smithi026.stdout:124: delete oid 44 current snap is 6 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:124: done (2 left) 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:125: rollback oid 23 current snap is 6 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 23 to 2 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:126: read oid 30 snap -1 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:126: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 6 2024-09-13T14:19:30.530 INFO:tasks.rados.rados.0.smithi026.stdout:127: seq_num 67 ranges {622592=729088,1351680=401408} 2024-09-13T14:19:30.538 INFO:tasks.rados.rados.0.smithi026.stdout:127: writing smithi02683976-38 from 622592 to 1351680 tid 1 2024-09-13T14:19:30.540 INFO:tasks.rados.rados.0.smithi026.stdout:127: writing smithi02683976-38 from 1351680 to 1753088 tid 2 2024-09-13T14:19:30.541 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 6 2024-09-13T14:19:30.541 INFO:tasks.rados.rados.0.smithi026.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-13T14:19:30.565 INFO:tasks.rados.rados.0.smithi026.stdout:126: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-13T14:19:30.580 INFO:tasks.rados.rados.0.smithi026.stdout:125: finishing rollback tid 0 to smithi02683976-23 2024-09-13T14:19:30.580 INFO:tasks.rados.rados.0.smithi026.stdout:125: finishing rollback tid 1 to smithi02683976-23 2024-09-13T14:19:30.580 INFO:tasks.rados.rados.0.smithi026.stdout:125: finishing rollback tid 2 to smithi02683976-23 2024-09-13T14:19:30.580 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 38 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-13T14:19:30.581 INFO:tasks.rados.rados.0.smithi026.stdout:123: expect (ObjNum 59 snap 4 seq_num 59) 2024-09-13T14:19:30.643 INFO:tasks.rados.rados.0.smithi026.stdout:127: finishing write tid 1 to smithi02683976-38 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:127: finishing write tid 2 to smithi02683976-38 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:127: finishing write tid 3 to smithi02683976-38 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:127: finishing write tid 4 to smithi02683976-38 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 42 (ObjNum 67 snap 6 seq_num 67) dirty exists 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:127: left oid 38 (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:19:30.644 INFO:tasks.rados.rados.0.smithi026.stdout:128: seq_num 68 ranges {573440=745472,1318912=401408} 2024-09-13T14:19:30.651 INFO:tasks.rados.rados.0.smithi026.stdout:128: writing smithi02683976-35 from 573440 to 1318912 tid 1 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:128: writing smithi02683976-35 from 1318912 to 1720320 tid 2 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:122: done (5 left) 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:123: done (4 left) 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:125: done (3 left) 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:126: done (2 left) 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:127: done (1 left) 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:129: copy_from oid 23 from oid 46 current snap is 6 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:130: copy_from oid 29 from oid 10 current snap is 6 2024-09-13T14:19:30.654 INFO:tasks.rados.rados.0.smithi026.stdout:131: snap_create 2024-09-13T14:19:30.655 INFO:tasks.rados.rados.0.smithi026.stdout:130: finishing copy_from to smithi02683976-29 2024-09-13T14:19:30.655 INFO:tasks.rados.rados.0.smithi026.stdout:130: got expected ENOENT (src dne) 2024-09-13T14:19:30.655 INFO:tasks.rados.rados.0.smithi026.stdout:130: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:19:30.655 INFO:tasks.rados.rados.0.smithi026.stdout:128: finishing write tid 1 to smithi02683976-35 2024-09-13T14:19:30.656 INFO:tasks.rados.rados.0.smithi026.stdout:128: finishing write tid 2 to smithi02683976-35 2024-09-13T14:19:30.656 INFO:tasks.rados.rados.0.smithi026.stdout:128: finishing write tid 3 to smithi02683976-35 2024-09-13T14:19:30.656 INFO:tasks.rados.rados.0.smithi026.stdout:128: finishing write tid 4 to smithi02683976-35 2024-09-13T14:19:30.657 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 37 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-13T14:19:30.657 INFO:tasks.rados.rados.0.smithi026.stdout:128: left oid 35 (ObjNum 68 snap 6 seq_num 68) 2024-09-13T14:19:30.658 INFO:tasks.rados.rados.0.smithi026.stdout:129: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:19:30.659 INFO:tasks.rados.rados.0.smithi026.stdout:129: finishing copy_from to smithi02683976-23 2024-09-13T14:19:30.659 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 4 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-13T14:19:30.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:30 smithi026 ceph-mon[27770]: pgmap v118: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:30 smithi026 ceph-mon[27770]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:19:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:30 smithi026 ceph-mon[39182]: pgmap v118: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:30 smithi026 ceph-mon[39182]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:19:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:30 smithi145 ceph-mon[31057]: pgmap v118: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:30 smithi145 ceph-mon[31057]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:19:31.529 INFO:tasks.rados.rados.0.smithi026.stdout:128: done (3 left) 2024-09-13T14:19:31.529 INFO:tasks.rados.rados.0.smithi026.stdout:129: done (2 left) 2024-09-13T14:19:31.529 INFO:tasks.rados.rados.0.smithi026.stdout:130: done (1 left) 2024-09-13T14:19:31.529 INFO:tasks.rados.rados.0.smithi026.stdout:131: done (0 left) 2024-09-13T14:19:31.529 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 7 2024-09-13T14:19:31.530 INFO:tasks.rados.rados.0.smithi026.stdout:132: seq_num 69 ranges {139264=761856,901120=401408} 2024-09-13T14:19:31.534 INFO:tasks.rados.rados.0.smithi026.stdout:132: writing smithi02683976-9 from 139264 to 901120 tid 1 2024-09-13T14:19:31.536 INFO:tasks.rados.rados.0.smithi026.stdout:132: writing smithi02683976-9 from 901120 to 1302528 tid 2 2024-09-13T14:19:31.536 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 7 2024-09-13T14:19:31.536 INFO:tasks.rados.rados.0.smithi026.stdout:133: seq_num 70 ranges {0=778240,778240=401408} 2024-09-13T14:19:31.541 INFO:tasks.rados.rados.0.smithi026.stdout:133: writing smithi02683976-12 from 0 to 778240 tid 1 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:133: writing smithi02683976-12 from 778240 to 1179648 tid 2 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:134: rollback oid 22 current snap is 7 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 0 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: finishing write tid 1 to smithi02683976-9 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: finishing write tid 2 to smithi02683976-9 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: finishing write tid 3 to smithi02683976-9 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: finishing write tid 4 to smithi02683976-9 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 27 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: left oid 9 (ObjNum 69 snap 7 seq_num 69) 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:132: done (2 left) 2024-09-13T14:19:31.544 INFO:tasks.rados.rados.0.smithi026.stdout:135: snap_create 2024-09-13T14:19:31.548 INFO:tasks.rados.rados.0.smithi026.stdout:133: finishing write tid 1 to smithi02683976-12 2024-09-13T14:19:31.548 INFO:tasks.rados.rados.0.smithi026.stdout:133: finishing write tid 2 to smithi02683976-12 2024-09-13T14:19:31.548 INFO:tasks.rados.rados.0.smithi026.stdout:133: finishing write tid 3 to smithi02683976-12 2024-09-13T14:19:31.551 INFO:tasks.rados.rados.0.smithi026.stdout:134: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:19:31.551 INFO:tasks.rados.rados.0.smithi026.stdout:133: finishing write tid 4 to smithi02683976-12 2024-09-13T14:19:31.551 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 12 (ObjNum 70 snap 7 seq_num 70) dirty exists 2024-09-13T14:19:31.551 INFO:tasks.rados.rados.0.smithi026.stdout:133: left oid 12 (ObjNum 70 snap 7 seq_num 70) 2024-09-13T14:19:31.551 INFO:tasks.rados.rados.0.smithi026.stdout:134: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:19:31.552 INFO:tasks.rados.rados.0.smithi026.stdout:134: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:19:31.552 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 16 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-13T14:19:31.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:31 smithi026 ceph-mon[27770]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:19:31.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:31 smithi026 ceph-mon[39182]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:19:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:31 smithi145 ceph-mon[31057]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:19:32.535 INFO:tasks.rados.rados.0.smithi026.stdout:133: done (2 left) 2024-09-13T14:19:32.535 INFO:tasks.rados.rados.0.smithi026.stdout:134: done (1 left) 2024-09-13T14:19:32.535 INFO:tasks.rados.rados.0.smithi026.stdout:135: done (0 left) 2024-09-13T14:19:32.535 INFO:tasks.rados.rados.0.smithi026.stdout:136: read oid 33 snap -1 2024-09-13T14:19:32.535 INFO:tasks.rados.rados.0.smithi026.stdout:136: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:137: read oid 45 snap -1 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:137: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:138: rmattr oid 43 current snap is 8 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:138: done (2 left) 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 8 2024-09-13T14:19:32.536 INFO:tasks.rados.rados.0.smithi026.stdout:139: seq_num 71 ranges {0=794624,794624=401408} 2024-09-13T14:19:32.539 INFO:tasks.rados.rados.0.smithi026.stdout:139: writing smithi02683976-13 from 0 to 794624 tid 1 2024-09-13T14:19:32.542 INFO:tasks.rados.rados.0.smithi026.stdout:139: writing smithi02683976-13 from 794624 to 1196032 tid 2 2024-09-13T14:19:32.542 INFO:tasks.rados.rados.0.smithi026.stdout:140: copy_from oid 43 from oid 26 current snap is 8 2024-09-13T14:19:32.542 INFO:tasks.rados.rados.0.smithi026.stdout:141: rollback oid 24 current snap is 8 2024-09-13T14:19:32.542 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 3 2024-09-13T14:19:32.542 INFO:tasks.rados.rados.0.smithi026.stdout:136: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:32.560 INFO:tasks.rados.rados.0.smithi026.stdout:137: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:141: finishing rollback tid 0 to smithi02683976-24 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:141: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:136: done (4 left) 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:137: done (3 left) 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:142: copy_from oid 2 from oid 22 current snap is 8 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:141: finishing rollback tid 2 to smithi02683976-24 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 32 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:139: finishing write tid 1 to smithi02683976-13 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:139: finishing write tid 2 to smithi02683976-13 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:139: finishing write tid 3 to smithi02683976-13 2024-09-13T14:19:32.586 INFO:tasks.rados.rados.0.smithi026.stdout:139: finishing write tid 4 to smithi02683976-13 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 28 (ObjNum 71 snap 8 seq_num 71) dirty exists 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:139: left oid 13 (ObjNum 71 snap 8 seq_num 71) 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:140: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:140: finishing copy_from to smithi02683976-43 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:139: done (3 left) 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:140: done (2 left) 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:141: done (1 left) 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:143: copy_from oid 34 from oid 45 current snap is 8 2024-09-13T14:19:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:144: delete oid 49 current snap is 8 2024-09-13T14:19:32.588 INFO:tasks.rados.rados.0.smithi026.stdout:144: done (2 left) 2024-09-13T14:19:32.588 INFO:tasks.rados.rados.0.smithi026.stdout:145: delete oid 42 current snap is 8 2024-09-13T14:19:32.589 INFO:tasks.rados.rados.0.smithi026.stdout:145: done (2 left) 2024-09-13T14:19:32.589 INFO:tasks.rados.rados.0.smithi026.stdout:146: rmattr oid 32 current snap is 8 2024-09-13T14:19:32.589 INFO:tasks.rados.rados.0.smithi026.stdout:146: done (2 left) 2024-09-13T14:19:32.589 INFO:tasks.rados.rados.0.smithi026.stdout:147: snap_remove snap 0 2024-09-13T14:19:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[27770]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[27770]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[39182]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[39182]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:32 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:32.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:32 smithi145 ceph-mon[31057]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.5 MiB/s wr, 89 op/s 2024-09-13T14:19:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:32 smithi145 ceph-mon[31057]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:32 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:33.542 INFO:tasks.rados.rados.0.smithi026.stdout:147: done (2 left) 2024-09-13T14:19:33.542 INFO:tasks.rados.rados.0.smithi026.stdout:148: read oid 8 snap -1 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:148: expect (ObjNum 7 snap 0 seq_num 7) 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:149: rmattr oid 12 current snap is 8 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:149: done (3 left) 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:150: snap_create 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:143: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:142: finishing copy_from racing read to smithi02683976-2 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:143: finishing copy_from to smithi02683976-34 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:142: finishing copy_from to smithi02683976-2 2024-09-13T14:19:33.543 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 16 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-13T14:19:33.545 INFO:tasks.rados.rados.0.smithi026.stdout:148: expect (ObjNum 7 snap 0 seq_num 7) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:142: done (3 left) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:143: done (2 left) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:148: done (1 left) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:150: done (0 left) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:151: copy_from oid 17 from oid 8 current snap is 9 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:152: read oid 32 snap -1 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:152: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-13T14:19:33.719 INFO:tasks.rados.rados.0.smithi026.stdout:153: snap_remove snap 6 2024-09-13T14:19:33.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:33 smithi026 ceph-mon[39182]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:19:33.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:33 smithi026 ceph-mon[27770]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:19:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:33 smithi145 ceph-mon[31057]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:19:34.724 INFO:tasks.rados.rados.0.smithi026.stdout:153: done (2 left) 2024-09-13T14:19:34.724 INFO:tasks.rados.rados.0.smithi026.stdout:154: rmattr oid 13 current snap is 9 2024-09-13T14:19:34.725 INFO:tasks.rados.rados.0.smithi026.stdout:154: done (2 left) 2024-09-13T14:19:34.725 INFO:tasks.rados.rados.0.smithi026.stdout:155: copy_from oid 27 from oid 29 current snap is 9 2024-09-13T14:19:34.725 INFO:tasks.rados.rados.0.smithi026.stdout:156: snap_remove snap 4 2024-09-13T14:19:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[27770]: pgmap v124: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-09-13T14:19:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[27770]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:19:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[27770]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:19:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[39182]: pgmap v124: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-09-13T14:19:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[39182]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:19:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:34 smithi026 ceph-mon[39182]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:19:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:34 smithi145 ceph-mon[31057]: pgmap v124: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-09-13T14:19:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:34 smithi145 ceph-mon[31057]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:19:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:34 smithi145 ceph-mon[31057]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:19:35.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:35] "GET /metrics HTTP/1.1" 200 239670 "" "Prometheus/2.43.0" 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:156: done (3 left) 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:157: read oid 14 snap -1 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:157: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:158: rollback oid 37 current snap is 9 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 8 2024-09-13T14:19:35.726 INFO:tasks.rados.rados.0.smithi026.stdout:152: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:151: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:151: finishing copy_from to smithi02683976-17 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 2 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:155: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:155: finishing copy_from to smithi02683976-27 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:158: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:158: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:151: done (4 left) 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:152: done (3 left) 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:155: done (2 left) 2024-09-13T14:19:35.747 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 9 2024-09-13T14:19:35.748 INFO:tasks.rados.rados.0.smithi026.stdout:158: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:19:35.748 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 20 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-13T14:19:35.748 INFO:tasks.rados.rados.0.smithi026.stdout:157: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-13T14:19:35.753 INFO:tasks.rados.rados.0.smithi026.stdout:159: seq_num 74 ranges {122880=442368,565248=696320,1261568=106496} 2024-09-13T14:19:35.756 INFO:tasks.rados.rados.0.smithi026.stdout:159: writing smithi02683976-17 from 122880 to 565248 tid 1 2024-09-13T14:19:35.760 INFO:tasks.rados.rados.0.smithi026.stdout:159: writing smithi02683976-17 from 565248 to 1261568 tid 2 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:159: writing smithi02683976-17 from 1261568 to 1368064 tid 3 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:157: done (2 left) 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:158: done (1 left) 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:160: read oid 35 snap -1 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:160: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-13T14:19:35.761 INFO:tasks.rados.rados.0.smithi026.stdout:161: rollback oid 37 current snap is 9 2024-09-13T14:19:35.762 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 3 2024-09-13T14:19:35.762 INFO:tasks.rados.rados.0.smithi026.stdout:159: finishing write tid 1 to smithi02683976-17 2024-09-13T14:19:35.762 INFO:tasks.rados.rados.0.smithi026.stdout:162: delete oid 32 current snap is 9 2024-09-13T14:19:35.764 INFO:tasks.rados.rados.0.smithi026.stdout:159: finishing write tid 2 to smithi02683976-17 2024-09-13T14:19:35.764 INFO:tasks.rados.rados.0.smithi026.stdout:159: finishing write tid 3 to smithi02683976-17 2024-09-13T14:19:35.764 INFO:tasks.rados.rados.0.smithi026.stdout:162: done (3 left) 2024-09-13T14:19:35.764 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 9 2024-09-13T14:19:35.764 INFO:tasks.rados.rados.0.smithi026.stdout:163: seq_num 76 ranges {794624=475136,1269760=458752,1728512=344064} 2024-09-13T14:19:35.771 INFO:tasks.rados.rados.0.smithi026.stdout:163: writing smithi02683976-2 from 794624 to 1269760 tid 1 2024-09-13T14:19:35.774 INFO:tasks.rados.rados.0.smithi026.stdout:163: writing smithi02683976-2 from 1269760 to 1728512 tid 2 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:163: writing smithi02683976-2 from 1728512 to 2072576 tid 3 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:164: snap_create 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:159: finishing write tid 4 to smithi02683976-17 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:161: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:159: finishing write tid 5 to smithi02683976-17 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 28 (ObjNum 74 snap 9 seq_num 74) dirty exists 2024-09-13T14:19:35.777 INFO:tasks.rados.rados.0.smithi026.stdout:159: left oid 17 (ObjNum 74 snap 9 seq_num 74) 2024-09-13T14:19:35.778 INFO:tasks.rados.rados.0.smithi026.stdout:161: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:19:35.778 INFO:tasks.rados.rados.0.smithi026.stdout:161: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:19:35.778 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 25 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-13T14:19:35.778 INFO:tasks.rados.rados.0.smithi026.stdout:160: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-13T14:19:35.837 INFO:tasks.rados.rados.0.smithi026.stdout:163: finishing write tid 1 to smithi02683976-2 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:163: finishing write tid 2 to smithi02683976-2 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:163: finishing write tid 3 to smithi02683976-2 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:163: finishing write tid 4 to smithi02683976-2 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:163: finishing write tid 5 to smithi02683976-2 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 57 (ObjNum 76 snap 9 seq_num 76) dirty exists 2024-09-13T14:19:35.838 INFO:tasks.rados.rados.0.smithi026.stdout:163: left oid 2 (ObjNum 76 snap 9 seq_num 76) 2024-09-13T14:19:36.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:35 smithi026 ceph-mon[27770]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:19:36.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:35 smithi026 ceph-mon[39182]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:19:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:35 smithi145 ceph-mon[31057]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:159: done (4 left) 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:160: done (3 left) 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:161: done (2 left) 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:163: done (1 left) 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:164: done (0 left) 2024-09-13T14:19:36.731 INFO:tasks.rados.rados.0.smithi026.stdout:165: snap_remove snap 8 2024-09-13T14:19:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:36 smithi026 ceph-mon[39182]: pgmap v128: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.4 MiB/s wr, 36 op/s 2024-09-13T14:19:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:36 smithi026 ceph-mon[39182]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:19:37.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:36 smithi026 ceph-mon[27770]: pgmap v128: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.4 MiB/s wr, 36 op/s 2024-09-13T14:19:37.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:36 smithi026 ceph-mon[27770]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:19:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:36 smithi145 ceph-mon[31057]: pgmap v128: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.4 MiB/s wr, 36 op/s 2024-09-13T14:19:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:36 smithi145 ceph-mon[31057]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:19:37.739 INFO:tasks.rados.rados.0.smithi026.stdout:165: done (0 left) 2024-09-13T14:19:37.739 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 10 2024-09-13T14:19:37.739 INFO:tasks.rados.rados.0.smithi026.stdout:166: seq_num 77 ranges {794624=491520,1286144=794624,2080768=8192} 2024-09-13T14:19:37.744 INFO:tasks.rados.rados.0.smithi026.stdout:166: writing smithi02683976-22 from 794624 to 1286144 tid 1 2024-09-13T14:19:37.749 INFO:tasks.rados.rados.0.smithi026.stdout:166: writing smithi02683976-22 from 1286144 to 2080768 tid 2 2024-09-13T14:19:37.749 INFO:tasks.rados.rados.0.smithi026.stdout:166: writing smithi02683976-22 from 2080768 to 2088960 tid 3 2024-09-13T14:19:37.749 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 10 2024-09-13T14:19:37.749 INFO:tasks.rados.rados.0.smithi026.stdout:166: finishing write tid 1 to smithi02683976-22 2024-09-13T14:19:37.749 INFO:tasks.rados.rados.0.smithi026.stdout:167: seq_num 78 ranges {0=516096,516096=679936,1196032=122880} 2024-09-13T14:19:37.752 INFO:tasks.rados.rados.0.smithi026.stdout:167: writing smithi02683976-10 from 0 to 516096 tid 1 2024-09-13T14:19:37.756 INFO:tasks.rados.rados.0.smithi026.stdout:167: writing smithi02683976-10 from 516096 to 1196032 tid 2 2024-09-13T14:19:37.757 INFO:tasks.rados.rados.0.smithi026.stdout:167: writing smithi02683976-10 from 1196032 to 1318912 tid 3 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:168: setattr oid 6 current snap is 10 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:166: finishing write tid 2 to smithi02683976-22 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:166: finishing write tid 3 to smithi02683976-22 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:166: finishing write tid 4 to smithi02683976-22 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:166: finishing write tid 5 to smithi02683976-22 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 29 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:166: left oid 22 (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:19:37.758 INFO:tasks.rados.rados.0.smithi026.stdout:167: finishing write tid 1 to smithi02683976-10 2024-09-13T14:19:37.759 INFO:tasks.rados.rados.0.smithi026.stdout:166: done (2 left) 2024-09-13T14:19:37.759 INFO:tasks.rados.rados.0.smithi026.stdout:169: read oid 12 snap 3 2024-09-13T14:19:37.759 INFO:tasks.rados.rados.0.smithi026.stdout:169: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-13T14:19:37.759 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 10 2024-09-13T14:19:37.759 INFO:tasks.rados.rados.0.smithi026.stdout:170: seq_num 79 ranges {950272=532480,1482752=565248,2048000=237568} 2024-09-13T14:19:37.767 INFO:tasks.rados.rados.0.smithi026.stdout:170: writing smithi02683976-3 from 950272 to 1482752 tid 1 2024-09-13T14:19:37.771 INFO:tasks.rados.rados.0.smithi026.stdout:170: writing smithi02683976-3 from 1482752 to 2048000 tid 2 2024-09-13T14:19:37.772 INFO:tasks.rados.rados.0.smithi026.stdout:170: writing smithi02683976-3 from 2048000 to 2285568 tid 3 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:171: snap_create 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:167: finishing write tid 2 to smithi02683976-10 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:167: finishing write tid 3 to smithi02683976-10 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:167: finishing write tid 4 to smithi02683976-10 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 37 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-13T14:19:37.773 INFO:tasks.rados.rados.0.smithi026.stdout:169: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:167: finishing write tid 5 to smithi02683976-10 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 41 (ObjNum 78 snap 10 seq_num 78) dirty exists 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:167: left oid 10 (ObjNum 78 snap 10 seq_num 78) 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:170: finishing write tid 1 to smithi02683976-3 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:170: finishing write tid 2 to smithi02683976-3 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:170: finishing write tid 3 to smithi02683976-3 2024-09-13T14:19:37.779 INFO:tasks.rados.rados.0.smithi026.stdout:170: finishing write tid 4 to smithi02683976-3 2024-09-13T14:19:37.780 INFO:tasks.rados.rados.0.smithi026.stdout:170: finishing write tid 5 to smithi02683976-3 2024-09-13T14:19:37.780 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 58 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:19:37.780 INFO:tasks.rados.rados.0.smithi026.stdout:170: left oid 3 (ObjNum 79 snap 10 seq_num 79) 2024-09-13T14:19:38.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:37 smithi026 ceph-mon[39182]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:19:38.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:37 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:38.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:37 smithi026 ceph-mon[27770]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:19:38.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:37 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:37 smithi145 ceph-mon[31057]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:19:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:37 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:167: done (4 left) 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:168: done (3 left) 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:169: done (2 left) 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:170: done (1 left) 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:171: done (0 left) 2024-09-13T14:19:38.743 INFO:tasks.rados.rados.0.smithi026.stdout:172: snap_create 2024-09-13T14:19:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:38 smithi026 ceph-mon[39182]: pgmap v131: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:38 smithi026 ceph-mon[39182]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:19:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:38 smithi026 ceph-mon[27770]: pgmap v131: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:38 smithi026 ceph-mon[27770]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:19:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:38 smithi145 ceph-mon[31057]: pgmap v131: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:38 smithi145 ceph-mon[31057]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:19:39.749 INFO:tasks.rados.rados.0.smithi026.stdout:172: done (0 left) 2024-09-13T14:19:39.749 INFO:tasks.rados.rados.0.smithi026.stdout:173: read oid 45 snap -1 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:173: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:174: read oid 11 snap -1 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:174: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:175: rollback oid 31 current snap is 12 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 11 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:176: read oid 44 snap -1 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:176: expect deleted 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:177: read oid 19 snap -1 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:177: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-13T14:19:39.750 INFO:tasks.rados.rados.0.smithi026.stdout:178: setattr oid 10 current snap is 12 2024-09-13T14:19:39.751 INFO:tasks.rados.rados.0.smithi026.stdout:179: snap_remove snap 3 2024-09-13T14:19:40.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:39 smithi026 ceph-mon[27770]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:19:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:39 smithi026 ceph-mon[39182]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:19:40.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:39 smithi145 ceph-mon[31057]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:19:40.754 INFO:tasks.rados.rados.0.smithi026.stdout:179: done (6 left) 2024-09-13T14:19:40.754 INFO:tasks.rados.rados.0.smithi026.stdout:180: read oid 8 snap 9 2024-09-13T14:19:40.754 INFO:tasks.rados.rados.0.smithi026.stdout:180: expect (ObjNum 7 snap 0 seq_num 7) 2024-09-13T14:19:40.754 INFO:tasks.rados.rados.0.smithi026.stdout:181: delete oid 32 current snap is 12 2024-09-13T14:19:40.754 INFO:tasks.rados.rados.0.smithi026.stdout:177: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:177: done (7 left) 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:181: done (6 left) 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:182: rollback oid 9 current snap is 12 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 11 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:183: delete oid 20 current snap is 12 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 43 (ObjNum 78 snap 10 seq_num 78) dirty exists 2024-09-13T14:19:40.770 INFO:tasks.rados.rados.0.smithi026.stdout:173: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:40.798 INFO:tasks.rados.rados.0.smithi026.stdout:175: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:19:40.798 INFO:tasks.rados.rados.0.smithi026.stdout:174: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:19:40.803 INFO:tasks.rados.rados.0.smithi026.stdout:175: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:19:40.803 INFO:tasks.rados.rados.0.smithi026.stdout:175: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:19:40.803 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 41 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-13T14:19:40.803 INFO:tasks.rados.rados.0.smithi026.stdout:180: expect (ObjNum 7 snap 0 seq_num 7) 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:182: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:182: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:182: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 51 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:173: done (7 left) 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:174: done (6 left) 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:175: done (5 left) 2024-09-13T14:19:40.808 INFO:tasks.rados.rados.0.smithi026.stdout:176: done (4 left) 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:178: done (3 left) 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:180: done (2 left) 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:182: done (1 left) 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:183: done (0 left) 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:184: copy_from oid 44 from oid 24 current snap is 12 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:185: rollback oid 13 current snap is 12 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 7 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 12 2024-09-13T14:19:40.809 INFO:tasks.rados.rados.0.smithi026.stdout:186: seq_num 82 ranges {1368064=581632,1949696=622592,2572288=180224} 2024-09-13T14:19:40.818 INFO:tasks.rados.rados.0.smithi026.stdout:186: writing smithi02683976-17 from 1368064 to 1949696 tid 1 2024-09-13T14:19:40.822 INFO:tasks.rados.rados.0.smithi026.stdout:186: writing smithi02683976-17 from 1949696 to 2572288 tid 2 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:186: writing smithi02683976-17 from 2572288 to 2752512 tid 3 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:187: rollback oid 35 current snap is 12 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 35 to 11 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:185: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:185: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:19:40.824 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 52 (ObjNum 1 snap 0 seq_num 304) dirty dne 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:184: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:184: finishing copy_from to smithi02683976-44 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 32 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:186: finishing write tid 1 to smithi02683976-17 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:184: done (3 left) 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:185: done (2 left) 2024-09-13T14:19:40.825 INFO:tasks.rados.rados.0.smithi026.stdout:188: snap_remove snap 5 2024-09-13T14:19:41.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:40 smithi026 ceph-mon[39182]: pgmap v134: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:41.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:40 smithi026 ceph-mon[39182]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:19:41.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:40 smithi026 ceph-mon[27770]: pgmap v134: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:41.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:40 smithi026 ceph-mon[27770]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:19:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:40 smithi145 ceph-mon[31057]: pgmap v134: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:40 smithi145 ceph-mon[31057]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:19:41.758 INFO:tasks.rados.rados.0.smithi026.stdout:188: done (2 left) 2024-09-13T14:19:41.758 INFO:tasks.rados.rados.0.smithi026.stdout:189: snap_remove snap 10 2024-09-13T14:19:42.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:41 smithi026 ceph-mon[27770]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:19:42.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:41 smithi026 ceph-mon[39182]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:19:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:41 smithi145 ceph-mon[31057]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:189: done (2 left) 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 12 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:186: finishing write tid 2 to smithi02683976-17 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:186: finishing write tid 3 to smithi02683976-17 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:187: finishing rollback tid 0 to smithi02683976-35 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:186: finishing write tid 4 to smithi02683976-17 2024-09-13T14:19:42.761 INFO:tasks.rados.rados.0.smithi026.stdout:190: seq_num 83 ranges {393216=598016,991232=499712,1490944=303104} 2024-09-13T14:19:42.768 INFO:tasks.rados.rados.0.smithi026.stdout:190: writing smithi02683976-24 from 393216 to 991232 tid 1 2024-09-13T14:19:42.771 INFO:tasks.rados.rados.0.smithi026.stdout:190: writing smithi02683976-24 from 991232 to 1490944 tid 2 2024-09-13T14:19:42.774 INFO:tasks.rados.rados.0.smithi026.stdout:190: writing smithi02683976-24 from 1490944 to 1794048 tid 3 2024-09-13T14:19:42.774 INFO:tasks.rados.rados.0.smithi026.stdout:191: copy_from oid 18 from oid 39 current snap is 12 2024-09-13T14:19:42.774 INFO:tasks.rados.rados.0.smithi026.stdout:187: finishing rollback tid 1 to smithi02683976-35 2024-09-13T14:19:42.774 INFO:tasks.rados.rados.0.smithi026.stdout:187: finishing rollback tid 2 to smithi02683976-35 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 68 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:186: finishing write tid 5 to smithi02683976-17 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 39 (ObjNum 82 snap 12 seq_num 82) dirty exists 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:186: left oid 17 (ObjNum 82 snap 12 seq_num 82) 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:190: finishing write tid 1 to smithi02683976-24 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:190: finishing write tid 2 to smithi02683976-24 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:186: done (3 left) 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:187: done (2 left) 2024-09-13T14:19:42.775 INFO:tasks.rados.rados.0.smithi026.stdout:192: delete oid 15 current snap is 12 2024-09-13T14:19:42.776 INFO:tasks.rados.rados.0.smithi026.stdout:192: done (2 left) 2024-09-13T14:19:42.776 INFO:tasks.rados.rados.0.smithi026.stdout:193: delete oid 41 current snap is 12 2024-09-13T14:19:42.776 INFO:tasks.rados.rados.0.smithi026.stdout:190: finishing write tid 3 to smithi02683976-24 2024-09-13T14:19:42.777 INFO:tasks.rados.rados.0.smithi026.stdout:190: finishing write tid 4 to smithi02683976-24 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:190: finishing write tid 5 to smithi02683976-24 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 76 (ObjNum 83 snap 12 seq_num 83) dirty exists 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:190: left oid 24 (ObjNum 83 snap 12 seq_num 83) 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:190: done (2 left) 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:193: done (1 left) 2024-09-13T14:19:42.779 INFO:tasks.rados.rados.0.smithi026.stdout:194: setattr oid 47 current snap is 12 2024-09-13T14:19:42.780 INFO:tasks.rados.rados.0.smithi026.stdout:195: snap_remove snap 2 2024-09-13T14:19:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:42 smithi026 ceph-mon[27770]: pgmap v137: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:42 smithi026 ceph-mon[27770]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:19:43.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:42 smithi026 ceph-mon[39182]: pgmap v137: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:43.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:42 smithi026 ceph-mon[39182]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:19:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:42 smithi145 ceph-mon[31057]: pgmap v137: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2024-09-13T14:19:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:42 smithi145 ceph-mon[31057]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:19:43.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:43.722 INFO:tasks.rados.rados.0.smithi026.stdout:195: done (2 left) 2024-09-13T14:19:43.722 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 12 2024-09-13T14:19:43.722 INFO:tasks.rados.rados.0.smithi026.stdout:196: seq_num 86 ranges {0=647168,647168=606208,1253376=196608} 2024-09-13T14:19:43.727 INFO:tasks.rados.rados.0.smithi026.stdout:196: writing smithi02683976-49 from 0 to 647168 tid 1 2024-09-13T14:19:43.732 INFO:tasks.rados.rados.0.smithi026.stdout:196: writing smithi02683976-49 from 647168 to 1253376 tid 2 2024-09-13T14:19:43.734 INFO:tasks.rados.rados.0.smithi026.stdout:196: writing smithi02683976-49 from 1253376 to 1449984 tid 3 2024-09-13T14:19:43.734 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 12 2024-09-13T14:19:43.734 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:43.735 INFO:tasks.rados.rados.0.smithi026.stdout:191: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:19:43.735 INFO:tasks.rados.rados.0.smithi026.stdout:191: finishing copy_from to smithi02683976-18 2024-09-13T14:19:43.735 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 22 (ObjNum 57 snap 4 seq_num 57) dirty exists 2024-09-13T14:19:43.735 INFO:tasks.rados.rados.0.smithi026.stdout:197: seq_num 87 ranges {221184=663552,884736=491520,1376256=311296} 2024-09-13T14:19:43.741 INFO:tasks.rados.rados.0.smithi026.stdout:197: writing smithi02683976-14 from 221184 to 884736 tid 1 2024-09-13T14:19:43.745 INFO:tasks.rados.rados.0.smithi026.stdout:197: writing smithi02683976-14 from 884736 to 1376256 tid 2 2024-09-13T14:19:43.747 INFO:tasks.rados.rados.0.smithi026.stdout:197: writing smithi02683976-14 from 1376256 to 1687552 tid 3 2024-09-13T14:19:43.748 INFO:tasks.rados.rados.0.smithi026.stdout:191: done (3 left) 2024-09-13T14:19:43.748 INFO:tasks.rados.rados.0.smithi026.stdout:194: done (2 left) 2024-09-13T14:19:43.748 INFO:tasks.rados.rados.0.smithi026.stdout:198: copy_from oid 27 from oid 26 current snap is 12 2024-09-13T14:19:43.748 INFO:tasks.rados.rados.0.smithi026.stdout:196: finishing write tid 1 to smithi02683976-49 2024-09-13T14:19:43.748 INFO:tasks.rados.rados.0.smithi026.stdout:196: finishing write tid 2 to smithi02683976-49 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:196: finishing write tid 3 to smithi02683976-49 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:196: finishing write tid 4 to smithi02683976-49 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:196: finishing write tid 5 to smithi02683976-49 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 55 (ObjNum 86 snap 12 seq_num 86) dirty exists 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:196: left oid 49 (ObjNum 86 snap 12 seq_num 86) 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:196: done (2 left) 2024-09-13T14:19:43.749 INFO:tasks.rados.rados.0.smithi026.stdout:199: snap_create 2024-09-13T14:19:43.750 INFO:tasks.rados.rados.0.smithi026.stdout:197: finishing write tid 1 to smithi02683976-14 2024-09-13T14:19:43.751 INFO:tasks.rados.rados.0.smithi026.stdout:197: finishing write tid 2 to smithi02683976-14 2024-09-13T14:19:43.752 INFO:tasks.rados.rados.0.smithi026.stdout:197: finishing write tid 3 to smithi02683976-14 2024-09-13T14:19:43.752 INFO:tasks.rados.rados.0.smithi026.stdout:197: finishing write tid 4 to smithi02683976-14 2024-09-13T14:19:43.753 INFO:tasks.rados.rados.0.smithi026.stdout:198: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:19:43.754 INFO:tasks.rados.rados.0.smithi026.stdout:197: finishing write tid 5 to smithi02683976-14 2024-09-13T14:19:43.754 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 81 (ObjNum 87 snap 12 seq_num 87) dirty exists 2024-09-13T14:19:43.754 INFO:tasks.rados.rados.0.smithi026.stdout:197: left oid 14 (ObjNum 87 snap 12 seq_num 87) 2024-09-13T14:19:43.754 INFO:tasks.rados.rados.0.smithi026.stdout:198: finishing copy_from to smithi02683976-27 2024-09-13T14:19:43.755 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-13T14:19:44.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:43 smithi026 ceph-mon[27770]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:19:44.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:43 smithi026 ceph-mon[27770]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:19:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:43 smithi026 ceph-mon[39182]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:19:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:43 smithi026 ceph-mon[39182]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:19:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:43 smithi145 ceph-mon[31057]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:19:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:43 smithi145 ceph-mon[31057]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:19:44.723 INFO:tasks.rados.rados.0.smithi026.stdout:197: done (2 left) 2024-09-13T14:19:44.723 INFO:tasks.rados.rados.0.smithi026.stdout:198: done (1 left) 2024-09-13T14:19:44.723 INFO:tasks.rados.rados.0.smithi026.stdout:199: done (0 left) 2024-09-13T14:19:44.723 INFO:tasks.rados.rados.0.smithi026.stdout:200: snap_create 2024-09-13T14:19:45.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:44 smithi026 ceph-mon[39182]: pgmap v140: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:45.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:44 smithi026 ceph-mon[39182]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:19:45.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:44 smithi026 ceph-mon[27770]: pgmap v140: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:45.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:44 smithi026 ceph-mon[27770]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:19:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:44 smithi145 ceph-mon[31057]: pgmap v140: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:44 smithi145 ceph-mon[31057]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:19:45.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:45] "GET /metrics HTTP/1.1" 200 239670 "" "Prometheus/2.43.0" 2024-09-13T14:19:45.726 INFO:tasks.rados.rados.0.smithi026.stdout:200: done (0 left) 2024-09-13T14:19:45.726 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 14 2024-09-13T14:19:45.726 INFO:tasks.rados.rados.0.smithi026.stdout:201: seq_num 88 ranges {2752512=679936,3432448=778240,4210688=24576} 2024-09-13T14:19:45.745 INFO:tasks.rados.rados.0.smithi026.stdout:201: writing smithi02683976-17 from 2752512 to 3432448 tid 1 2024-09-13T14:19:45.749 INFO:tasks.rados.rados.0.smithi026.stdout:201: writing smithi02683976-17 from 3432448 to 4210688 tid 2 2024-09-13T14:19:45.749 INFO:tasks.rados.rados.0.smithi026.stdout:201: writing smithi02683976-17 from 4210688 to 4235264 tid 3 2024-09-13T14:19:45.750 INFO:tasks.rados.rados.0.smithi026.stdout:202: rollback oid 8 current snap is 14 2024-09-13T14:19:45.750 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 11 2024-09-13T14:19:45.750 INFO:tasks.rados.rados.0.smithi026.stdout:203: rmattr oid 26 current snap is 14 2024-09-13T14:19:45.750 INFO:tasks.rados.rados.0.smithi026.stdout:203: done (2 left) 2024-09-13T14:19:45.750 INFO:tasks.rados.rados.0.smithi026.stdout:204: delete oid 2 current snap is 14 2024-09-13T14:19:45.751 INFO:tasks.rados.rados.0.smithi026.stdout:201: finishing write tid 1 to smithi02683976-17 2024-09-13T14:19:45.752 INFO:tasks.rados.rados.0.smithi026.stdout:204: done (2 left) 2024-09-13T14:19:45.752 INFO:tasks.rados.rados.0.smithi026.stdout:205: rmattr oid 24 current snap is 14 2024-09-13T14:19:45.752 INFO:tasks.rados.rados.0.smithi026.stdout:205: done (2 left) 2024-09-13T14:19:45.752 INFO:tasks.rados.rados.0.smithi026.stdout:206: snap_create 2024-09-13T14:19:45.753 INFO:tasks.rados.rados.0.smithi026.stdout:202: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:19:45.754 INFO:tasks.rados.rados.0.smithi026.stdout:202: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:19:45.754 INFO:tasks.rados.rados.0.smithi026.stdout:202: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:19:45.754 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 6 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-13T14:19:45.754 INFO:tasks.rados.rados.0.smithi026.stdout:201: finishing write tid 2 to smithi02683976-17 2024-09-13T14:19:45.755 INFO:tasks.rados.rados.0.smithi026.stdout:201: finishing write tid 3 to smithi02683976-17 2024-09-13T14:19:45.755 INFO:tasks.rados.rados.0.smithi026.stdout:201: finishing write tid 4 to smithi02683976-17 2024-09-13T14:19:45.756 INFO:tasks.rados.rados.0.smithi026.stdout:201: finishing write tid 5 to smithi02683976-17 2024-09-13T14:19:45.756 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 58 (ObjNum 88 snap 14 seq_num 88) dirty exists 2024-09-13T14:19:45.756 INFO:tasks.rados.rados.0.smithi026.stdout:201: left oid 17 (ObjNum 88 snap 14 seq_num 88) 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:201: done (2 left) 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:202: done (1 left) 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:206: done (0 left) 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:207: copy_from oid 48 from oid 47 current snap is 15 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:208: read oid 34 snap -1 2024-09-13T14:19:46.730 INFO:tasks.rados.rados.0.smithi026.stdout:208: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:209: read oid 49 snap -1 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:209: expect (ObjNum 86 snap 12 seq_num 86) 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:210: read oid 41 snap -1 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:210: expect deleted 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:211: read oid 31 snap -1 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:211: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-13T14:19:46.731 INFO:tasks.rados.rados.0.smithi026.stdout:212: snap_remove snap 14 2024-09-13T14:19:46.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:46 smithi026 ceph-mon[27770]: pgmap v143: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:46.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:46 smithi026 ceph-mon[27770]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:19:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:46 smithi026 ceph-mon[39182]: pgmap v143: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:46 smithi026 ceph-mon[39182]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:19:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:46 smithi145 ceph-mon[31057]: pgmap v143: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1003 KiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-13T14:19:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:46 smithi145 ceph-mon[31057]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:19:47.735 INFO:tasks.rados.rados.0.smithi026.stdout:212: done (5 left) 2024-09-13T14:19:47.735 INFO:tasks.rados.rados.0.smithi026.stdout:213: copy_from oid 15 from oid 4 current snap is 15 2024-09-13T14:19:47.735 INFO:tasks.rados.rados.0.smithi026.stdout:211: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-13T14:19:47.749 INFO:tasks.rados.rados.0.smithi026.stdout:207: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:19:47.749 INFO:tasks.rados.rados.0.smithi026.stdout:209: expect (ObjNum 86 snap 12 seq_num 86) 2024-09-13T14:19:47.787 INFO:tasks.rados.rados.0.smithi026.stdout:207: finishing copy_from to smithi02683976-48 2024-09-13T14:19:47.787 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:47.787 INFO:tasks.rados.rados.0.smithi026.stdout:208: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:213: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:213: finishing copy_from to smithi02683976-15 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 14 (ObjNum 58 snap 4 seq_num 58) dirty exists 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:207: done (5 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:208: done (4 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:209: done (3 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:210: done (2 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:211: done (1 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:213: done (0 left) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:214: read oid 27 snap -1 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:214: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-13T14:19:47.811 INFO:tasks.rados.rados.0.smithi026.stdout:215: snap_remove snap 13 2024-09-13T14:19:47.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:47 smithi026 ceph-mon[39182]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:19:47.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:47 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:47.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:47 smithi026 ceph-mon[27770]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:19:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:47 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:47 smithi145 ceph-mon[31057]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:19:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:47 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:48.752 INFO:tasks.rados.rados.0.smithi026.stdout:215: done (1 left) 2024-09-13T14:19:48.753 INFO:tasks.rados.rados.0.smithi026.stdout:216: read oid 33 snap -1 2024-09-13T14:19:48.753 INFO:tasks.rados.rados.0.smithi026.stdout:216: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:48.753 INFO:tasks.rados.rados.0.smithi026.stdout:217: delete oid 36 current snap is 15 2024-09-13T14:19:48.753 INFO:tasks.rados.rados.0.smithi026.stdout:214: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-13T14:19:48.766 INFO:tasks.rados.rados.0.smithi026.stdout:216: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-13T14:19:48.781 INFO:tasks.rados.rados.0.smithi026.stdout:214: done (2 left) 2024-09-13T14:19:48.781 INFO:tasks.rados.rados.0.smithi026.stdout:216: done (1 left) 2024-09-13T14:19:48.781 INFO:tasks.rados.rados.0.smithi026.stdout:217: done (0 left) 2024-09-13T14:19:48.781 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 15 2024-09-13T14:19:48.781 INFO:tasks.rados.rados.0.smithi026.stdout:218: seq_num 91 ranges {491520=729088,1220608=425984,1646592=376832} 2024-09-13T14:19:48.789 INFO:tasks.rados.rados.0.smithi026.stdout:218: writing smithi02683976-30 from 491520 to 1220608 tid 1 2024-09-13T14:19:48.792 INFO:tasks.rados.rados.0.smithi026.stdout:218: writing smithi02683976-30 from 1220608 to 1646592 tid 2 2024-09-13T14:19:48.795 INFO:tasks.rados.rados.0.smithi026.stdout:218: writing smithi02683976-30 from 1646592 to 2023424 tid 3 2024-09-13T14:19:48.795 INFO:tasks.rados.rados.0.smithi026.stdout:219: rmattr oid 17 current snap is 15 2024-09-13T14:19:48.795 INFO:tasks.rados.rados.0.smithi026.stdout:219: done (1 left) 2024-09-13T14:19:48.796 INFO:tasks.rados.rados.0.smithi026.stdout:220: snap_remove snap 11 2024-09-13T14:19:48.909 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:19:49.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:48 smithi026 ceph-mon[39182]: pgmap v146: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:49.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:48 smithi026 ceph-mon[39182]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:19:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:48 smithi026 ceph-mon[27770]: pgmap v146: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:49.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:48 smithi026 ceph-mon[27770]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:19:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:48 smithi145 ceph-mon[31057]: pgmap v146: 121 pgs: 121 active+clean; 51 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:48 smithi145 ceph-mon[31057]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 116s ago 3m 15.7M - 0.25.0 c8568f914cd2 52eac4b9b3e5 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (3m) 2m ago 3m 82.6M - 9.4.7 954c08fa6188 b034aa056559 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (4m) 116s ago 4m 12.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (4m) 2m ago 4m 17.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7926da3ac6f2 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283,8765 running (9m) 116s ago 9m 446M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 90a812e595d2 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (9m) 116s ago 9m 47.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (8m) 2m ago 8m 40.8M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:19:49.438 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (8m) 116s ago 8m 39.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 116s ago 2m 13.4M - 1.5.0 0da6a335fe13 0720764634c2 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (2m) 2m ago 2m 3355k - 1.5.0 0da6a335fe13 02ae27c8e5f5 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (7m) 116s ago 7m 44.3M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (7m) 116s ago 7m 43.6M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 116s ago 7m 44.2M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (6m) 116s ago 6m 41.7M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (6m) 2m ago 6m 43.9M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (5m) 2m ago 5m 42.2M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (5m) 2m ago 5m 41.5M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (4m) 2m ago 4m 44.0M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:19:49.439 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (2m) 2m ago 3m 29.3M - 2.43.0 a07b618ecd1d 33771513b4fc 2024-09-13T14:19:49.754 INFO:tasks.rados.rados.0.smithi026.stdout:220: done (1 left) 2024-09-13T14:19:49.754 INFO:tasks.rados.rados.0.smithi026.stdout:221: snap_remove snap 7 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:19:49.763 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:19:49.764 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:19:50.037 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:19:50.037 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:19:50.037 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:19:50.037 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:19:50.037 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-09-13T14:19:50.038 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/20 daemons upgraded", 2024-09-13T14:19:50.038 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f image on host smithi145", 2024-09-13T14:19:50.038 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:19:50.038 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:19:50.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[27770]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:19:50.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:50.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[39182]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:19:50.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:50.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:49 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:49 smithi145 ceph-mon[31057]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:19:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:49 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:49 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:49 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:19:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:221: done (1 left) 2024-09-13T14:19:50.762 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 15 2024-09-13T14:19:50.762 INFO:tasks.rados.rados.0.smithi026.stdout:222: seq_num 92 ranges {90112=745472,835584=761856,1597440=40960} 2024-09-13T14:19:50.767 INFO:tasks.rados.rados.0.smithi026.stdout:222: writing smithi02683976-6 from 90112 to 835584 tid 1 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:222: writing smithi02683976-6 from 835584 to 1597440 tid 2 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:222: writing smithi02683976-6 from 1597440 to 1638400 tid 3 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:223: read oid 11 snap -1 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:223: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 15 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:218: finishing write tid 1 to smithi02683976-30 2024-09-13T14:19:50.772 INFO:tasks.rados.rados.0.smithi026.stdout:224: seq_num 93 ranges {458752=761856,1220608=647168,1867776=155648} 2024-09-13T14:19:50.779 INFO:tasks.rados.rados.0.smithi026.stdout:224: writing smithi02683976-28 from 458752 to 1220608 tid 1 2024-09-13T14:19:50.783 INFO:tasks.rados.rados.0.smithi026.stdout:224: writing smithi02683976-28 from 1220608 to 1867776 tid 2 2024-09-13T14:19:50.784 INFO:tasks.rados.rados.0.smithi026.stdout:224: writing smithi02683976-28 from 1867776 to 2023424 tid 3 2024-09-13T14:19:50.784 INFO:tasks.rados.rados.0.smithi026.stdout:225: snap_create 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:218: finishing write tid 2 to smithi02683976-30 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:218: finishing write tid 3 to smithi02683976-30 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:218: finishing write tid 4 to smithi02683976-30 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:218: finishing write tid 5 to smithi02683976-30 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 44 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:218: left oid 30 (ObjNum 91 snap 15 seq_num 91) 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: finishing write tid 1 to smithi02683976-6 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: finishing write tid 2 to smithi02683976-6 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: finishing write tid 3 to smithi02683976-6 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: finishing write tid 4 to smithi02683976-6 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: finishing write tid 5 to smithi02683976-6 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 80 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:222: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2024-09-13T14:19:50.785 INFO:tasks.rados.rados.0.smithi026.stdout:223: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:19:50.791 INFO:tasks.rados.rados.0.smithi026.stdout:224: finishing write tid 1 to smithi02683976-28 2024-09-13T14:19:50.791 INFO:tasks.rados.rados.0.smithi026.stdout:224: finishing write tid 2 to smithi02683976-28 2024-09-13T14:19:50.791 INFO:tasks.rados.rados.0.smithi026.stdout:224: finishing write tid 3 to smithi02683976-28 2024-09-13T14:19:50.791 INFO:tasks.rados.rados.0.smithi026.stdout:224: finishing write tid 4 to smithi02683976-28 2024-09-13T14:19:50.791 INFO:tasks.rados.rados.0.smithi026.stdout:224: finishing write tid 5 to smithi02683976-28 2024-09-13T14:19:50.792 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 68 (ObjNum 93 snap 15 seq_num 93) dirty exists 2024-09-13T14:19:50.792 INFO:tasks.rados.rados.0.smithi026.stdout:224: left oid 28 (ObjNum 93 snap 15 seq_num 93) 2024-09-13T14:19:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:19:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: from='client.24851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi145 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: pgmap v149: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1111906088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:19:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: from='client.24851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi145 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: pgmap v149: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:19:51.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:50 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1111906088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: from='client.24851 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi145 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: pgmap v149: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:19:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:50 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1111906088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:218: done (4 left) 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:222: done (3 left) 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:223: done (2 left) 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:224: done (1 left) 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:225: done (0 left) 2024-09-13T14:19:51.766 INFO:tasks.rados.rados.0.smithi026.stdout:226: delete oid 41 current snap is 16 2024-09-13T14:19:51.767 INFO:tasks.rados.rados.0.smithi026.stdout:226: done (0 left) 2024-09-13T14:19:51.767 INFO:tasks.rados.rados.0.smithi026.stdout:227: read oid 13 snap -1 2024-09-13T14:19:51.767 INFO:tasks.rados.rados.0.smithi026.stdout:227: expect deleted 2024-09-13T14:19:51.767 INFO:tasks.rados.rados.0.smithi026.stdout:228: setattr oid 26 current snap is 16 2024-09-13T14:19:51.769 INFO:tasks.rados.rados.0.smithi026.stdout:229: copy_from oid 32 from oid 7 current snap is 16 2024-09-13T14:19:51.769 INFO:tasks.rados.rados.0.smithi026.stdout:230: read oid 22 snap -1 2024-09-13T14:19:51.769 INFO:tasks.rados.rados.0.smithi026.stdout:230: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:19:51.769 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 16 2024-09-13T14:19:51.769 INFO:tasks.rados.rados.0.smithi026.stdout:231: seq_num 95 ranges {1032192=794624,1826816=409600,2236416=393216} 2024-09-13T14:19:51.779 INFO:tasks.rados.rados.0.smithi026.stdout:231: writing smithi02683976-15 from 1032192 to 1826816 tid 1 2024-09-13T14:19:51.782 INFO:tasks.rados.rados.0.smithi026.stdout:231: writing smithi02683976-15 from 1826816 to 2236416 tid 2 2024-09-13T14:19:51.784 INFO:tasks.rados.rados.0.smithi026.stdout:231: writing smithi02683976-15 from 2236416 to 2629632 tid 3 2024-09-13T14:19:51.784 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 16 2024-09-13T14:19:51.785 INFO:tasks.rados.rados.0.smithi026.stdout:232: seq_num 96 ranges {1449984=409600,1859584=696320,2555904=507904} 2024-09-13T14:19:51.795 INFO:tasks.rados.rados.0.smithi026.stdout:232: writing smithi02683976-49 from 1449984 to 1859584 tid 1 2024-09-13T14:19:51.799 INFO:tasks.rados.rados.0.smithi026.stdout:232: writing smithi02683976-49 from 1859584 to 2555904 tid 2 2024-09-13T14:19:51.802 INFO:tasks.rados.rados.0.smithi026.stdout:232: writing smithi02683976-49 from 2555904 to 3063808 tid 3 2024-09-13T14:19:51.802 INFO:tasks.rados.rados.0.smithi026.stdout:233: rollback oid 33 current snap is 16 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 9 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:229: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 43 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:229: finishing copy_from to smithi02683976-32 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:227: done (6 left) 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:228: done (5 left) 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:229: done (4 left) 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 16 2024-09-13T14:19:51.803 INFO:tasks.rados.rados.0.smithi026.stdout:234: seq_num 97 ranges {303104=434176,737280=581632,1318912=540672,1859584=81920} 2024-09-13T14:19:51.806 INFO:tasks.rados.rados.0.smithi026.stdout:234: writing smithi02683976-19 from 303104 to 737280 tid 1 2024-09-13T14:19:51.810 INFO:tasks.rados.rados.0.smithi026.stdout:234: writing smithi02683976-19 from 737280 to 1318912 tid 2 2024-09-13T14:19:51.813 INFO:tasks.rados.rados.0.smithi026.stdout:234: writing smithi02683976-19 from 1318912 to 1859584 tid 3 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:234: writing smithi02683976-19 from 1859584 to 1941504 tid 4 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:235: read oid 21 snap -1 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:235: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:236: rollback oid 45 current snap is 16 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 9 2024-09-13T14:19:51.814 INFO:tasks.rados.rados.0.smithi026.stdout:230: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:19:51.884 INFO:tasks.rados.rados.0.smithi026.stdout:231: finishing write tid 1 to smithi02683976-15 2024-09-13T14:19:51.884 INFO:tasks.rados.rados.0.smithi026.stdout:231: finishing write tid 2 to smithi02683976-15 2024-09-13T14:19:51.884 INFO:tasks.rados.rados.0.smithi026.stdout:231: finishing write tid 3 to smithi02683976-15 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:231: finishing write tid 4 to smithi02683976-15 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:231: finishing write tid 5 to smithi02683976-15 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 57 (ObjNum 95 snap 16 seq_num 95) dirty exists 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:231: left oid 15 (ObjNum 95 snap 16 seq_num 95) 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:232: finishing write tid 1 to smithi02683976-49 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:232: finishing write tid 2 to smithi02683976-49 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:232: finishing write tid 3 to smithi02683976-49 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:232: finishing write tid 4 to smithi02683976-49 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:233: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:233: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:230: done (6 left) 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:231: done (5 left) 2024-09-13T14:19:51.885 INFO:tasks.rados.rados.0.smithi026.stdout:237: rmattr oid 41 current snap is 16 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:237: done (5 left) 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:238: copy_from oid 46 from oid 14 current snap is 16 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:233: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 106 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:232: finishing write tid 5 to smithi02683976-49 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 73 (ObjNum 96 snap 16 seq_num 96) dirty exists 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:232: left oid 49 (ObjNum 96 snap 16 seq_num 96) 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 1 to smithi02683976-19 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 2 to smithi02683976-19 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 3 to smithi02683976-19 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:236: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 4 to smithi02683976-19 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:236: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:236: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:19:51.886 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 16 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:51.887 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 5 to smithi02683976-19 2024-09-13T14:19:51.887 INFO:tasks.rados.rados.0.smithi026.stdout:234: finishing write tid 6 to smithi02683976-19 2024-09-13T14:19:51.887 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 78 (ObjNum 97 snap 16 seq_num 97) dirty exists 2024-09-13T14:19:51.887 INFO:tasks.rados.rados.0.smithi026.stdout:234: left oid 19 (ObjNum 97 snap 16 seq_num 97) 2024-09-13T14:19:51.887 INFO:tasks.rados.rados.0.smithi026.stdout:235: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-13T14:19:51.894 INFO:tasks.rados.rados.0.smithi026.stdout:238: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:19:51.894 INFO:tasks.rados.rados.0.smithi026.stdout:232: done (5 left) 2024-09-13T14:19:51.894 INFO:tasks.rados.rados.0.smithi026.stdout:233: done (4 left) 2024-09-13T14:19:51.894 INFO:tasks.rados.rados.0.smithi026.stdout:234: done (3 left) 2024-09-13T14:19:51.894 INFO:tasks.rados.rados.0.smithi026.stdout:235: done (2 left) 2024-09-13T14:19:51.895 INFO:tasks.rados.rados.0.smithi026.stdout:236: done (1 left) 2024-09-13T14:19:51.895 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 16 2024-09-13T14:19:51.895 INFO:tasks.rados.rados.0.smithi026.stdout:239: seq_num 98 ranges {0=450560,450560=466944,917504=737280} 2024-09-13T14:19:51.897 INFO:tasks.rados.rados.0.smithi026.stdout:239: writing smithi02683976-36 from 0 to 450560 tid 1 2024-09-13T14:19:51.899 INFO:tasks.rados.rados.0.smithi026.stdout:239: writing smithi02683976-36 from 450560 to 917504 tid 2 2024-09-13T14:19:51.904 INFO:tasks.rados.rados.0.smithi026.stdout:239: writing smithi02683976-36 from 917504 to 1654784 tid 3 2024-09-13T14:19:51.904 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 16 2024-09-13T14:19:51.905 INFO:tasks.rados.rados.0.smithi026.stdout:240: seq_num 99 ranges {393216=466944,860160=802816,1662976=401408} 2024-09-13T14:19:51.908 INFO:tasks.rados.rados.0.smithi026.stdout:240: writing smithi02683976-44 from 393216 to 860160 tid 1 2024-09-13T14:19:51.913 INFO:tasks.rados.rados.0.smithi026.stdout:240: writing smithi02683976-44 from 860160 to 1662976 tid 2 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:240: writing smithi02683976-44 from 1662976 to 2064384 tid 3 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:241: read oid 38 snap 12 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:241: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:242: rmattr oid 49 current snap is 16 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:242: done (4 left) 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 16 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:238: finishing copy_from to smithi02683976-46 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 81 (ObjNum 87 snap 12 seq_num 87) dirty exists 2024-09-13T14:19:51.916 INFO:tasks.rados.rados.0.smithi026.stdout:243: seq_num 100 ranges {1032192=483328,1515520=688128,2203648=516096} 2024-09-13T14:19:51.924 INFO:tasks.rados.rados.0.smithi026.stdout:243: writing smithi02683976-4 from 1032192 to 1515520 tid 1 2024-09-13T14:19:51.928 INFO:tasks.rados.rados.0.smithi026.stdout:243: writing smithi02683976-4 from 1515520 to 2203648 tid 2 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:243: writing smithi02683976-4 from 2203648 to 2719744 tid 3 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:238: done (4 left) 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:244: read oid 19 snap -1 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:244: expect (ObjNum 97 snap 16 seq_num 97) 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:245: read oid 7 snap 9 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:245: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 16 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:239: finishing write tid 1 to smithi02683976-36 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:239: finishing write tid 2 to smithi02683976-36 2024-09-13T14:19:51.932 INFO:tasks.rados.rados.0.smithi026.stdout:239: finishing write tid 3 to smithi02683976-36 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:239: finishing write tid 4 to smithi02683976-36 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:239: finishing write tid 5 to smithi02683976-36 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 110 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:239: left oid 36 (ObjNum 98 snap 16 seq_num 98) 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:240: finishing write tid 1 to smithi02683976-44 2024-09-13T14:19:51.933 INFO:tasks.rados.rados.0.smithi026.stdout:246: seq_num 101 ranges {0=499712,499712=573440,1073152=524288,1597440=106496} 2024-09-13T14:19:51.935 INFO:tasks.rados.rados.0.smithi026.stdout:246: writing smithi02683976-42 from 0 to 499712 tid 1 2024-09-13T14:19:51.938 INFO:tasks.rados.rados.0.smithi026.stdout:246: writing smithi02683976-42 from 499712 to 1073152 tid 2 2024-09-13T14:19:51.942 INFO:tasks.rados.rados.0.smithi026.stdout:246: writing smithi02683976-42 from 1073152 to 1597440 tid 3 2024-09-13T14:19:51.942 INFO:tasks.rados.rados.0.smithi026.stdout:246: writing smithi02683976-42 from 1597440 to 1703936 tid 4 2024-09-13T14:19:51.943 INFO:tasks.rados.rados.0.smithi026.stdout:239: done (6 left) 2024-09-13T14:19:51.943 INFO:tasks.rados.rados.0.smithi026.stdout:247: snap_remove snap 15 2024-09-13T14:19:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:51 smithi026 ceph-mon[39182]: from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:51 smithi026 ceph-mon[39182]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:19:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:51 smithi026 ceph-mon[27770]: from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:51 smithi026 ceph-mon[27770]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:19:52.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:51 smithi145 ceph-mon[31057]: from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:52.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:51 smithi145 ceph-mon[31057]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:247: done (6 left) 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 16 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:240: finishing write tid 2 to smithi02683976-44 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:240: finishing write tid 3 to smithi02683976-44 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:240: finishing write tid 4 to smithi02683976-44 2024-09-13T14:19:52.772 INFO:tasks.rados.rados.0.smithi026.stdout:240: finishing write tid 5 to smithi02683976-44 2024-09-13T14:19:52.773 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 85 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-09-13T14:19:52.773 INFO:tasks.rados.rados.0.smithi026.stdout:240: left oid 44 (ObjNum 99 snap 16 seq_num 99) 2024-09-13T14:19:52.773 INFO:tasks.rados.rados.0.smithi026.stdout:241: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:19:52.830 INFO:tasks.rados.rados.0.smithi026.stdout:243: finishing write tid 1 to smithi02683976-4 2024-09-13T14:19:52.830 INFO:tasks.rados.rados.0.smithi026.stdout:243: finishing write tid 2 to smithi02683976-4 2024-09-13T14:19:52.830 INFO:tasks.rados.rados.0.smithi026.stdout:245: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:19:52.832 INFO:tasks.rados.rados.0.smithi026.stdout:243: finishing write tid 3 to smithi02683976-4 2024-09-13T14:19:52.833 INFO:tasks.rados.rados.0.smithi026.stdout:243: finishing write tid 4 to smithi02683976-4 2024-09-13T14:19:52.833 INFO:tasks.rados.rados.0.smithi026.stdout:243: finishing write tid 5 to smithi02683976-4 2024-09-13T14:19:52.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 62 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-09-13T14:19:52.833 INFO:tasks.rados.rados.0.smithi026.stdout:243: left oid 4 (ObjNum 100 snap 16 seq_num 100) 2024-09-13T14:19:52.833 INFO:tasks.rados.rados.0.smithi026.stdout:244: expect (ObjNum 97 snap 16 seq_num 97) 2024-09-13T14:19:52.888 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 1 to smithi02683976-42 2024-09-13T14:19:52.888 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 2 to smithi02683976-42 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 3 to smithi02683976-42 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 4 to smithi02683976-42 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 5 to smithi02683976-42 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:246: finishing write tid 6 to smithi02683976-42 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 48 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:246: left oid 42 (ObjNum 101 snap 16 seq_num 101) 2024-09-13T14:19:52.889 INFO:tasks.rados.rados.0.smithi026.stdout:248: seq_num 102 ranges {0=516096,516096=450560,966656=729088,1695744=24576} 2024-09-13T14:19:52.891 INFO:tasks.rados.rados.0.smithi026.stdout:248: writing smithi02683976-41 from 0 to 516096 tid 1 2024-09-13T14:19:52.894 INFO:tasks.rados.rados.0.smithi026.stdout:248: writing smithi02683976-41 from 516096 to 966656 tid 2 2024-09-13T14:19:52.899 INFO:tasks.rados.rados.0.smithi026.stdout:248: writing smithi02683976-41 from 966656 to 1695744 tid 3 2024-09-13T14:19:52.899 INFO:tasks.rados.rados.0.smithi026.stdout:248: writing smithi02683976-41 from 1695744 to 1720320 tid 4 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:240: done (6 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:241: done (5 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:243: done (4 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:244: done (3 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:245: done (2 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:246: done (1 left) 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:249: rollback oid 29 current snap is 16 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 9 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 1 to smithi02683976-41 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 2 to smithi02683976-41 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:250: rollback oid 12 current snap is 16 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 12 2024-09-13T14:19:52.900 INFO:tasks.rados.rados.0.smithi026.stdout:251: rmattr oid 46 current snap is 16 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:251: done (3 left) 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:252: rollback oid 19 current snap is 16 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 9 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:253: rmattr oid 3 current snap is 16 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:253: done (4 left) 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:254: copy_from oid 21 from oid 39 current snap is 16 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:255: rollback oid 7 current snap is 16 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 7 to 9 2024-09-13T14:19:52.901 INFO:tasks.rados.rados.0.smithi026.stdout:256: snap_create 2024-09-13T14:19:52.902 INFO:tasks.rados.rados.0.smithi026.stdout:255: finishing rollback tid 0 to smithi02683976-7 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:249: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:250: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:255: finishing rollback tid 1 to smithi02683976-7 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:250: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 3 to smithi02683976-41 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 4 to smithi02683976-41 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:250: finishing rollback tid 2 to smithi02683976-12 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 49 (ObjNum 70 snap 7 seq_num 70) dirty exists 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:249: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:249: finishing rollback tid 2 to smithi02683976-29 2024-09-13T14:19:52.903 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 84 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:19:52.904 INFO:tasks.rados.rados.0.smithi026.stdout:252: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:19:52.904 INFO:tasks.rados.rados.0.smithi026.stdout:255: finishing rollback tid 2 to smithi02683976-7 2024-09-13T14:19:52.904 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 6 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:19:52.904 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 5 to smithi02683976-41 2024-09-13T14:19:52.905 INFO:tasks.rados.rados.0.smithi026.stdout:252: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:19:52.905 INFO:tasks.rados.rados.0.smithi026.stdout:252: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:19:52.905 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 87 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-13T14:19:52.905 INFO:tasks.rados.rados.0.smithi026.stdout:248: finishing write tid 6 to smithi02683976-41 2024-09-13T14:19:52.905 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 76 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-13T14:19:52.906 INFO:tasks.rados.rados.0.smithi026.stdout:248: left oid 41 (ObjNum 102 snap 16 seq_num 102) 2024-09-13T14:19:52.909 INFO:tasks.rados.rados.0.smithi026.stdout:254: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:19:52.910 INFO:tasks.rados.rados.0.smithi026.stdout:254: finishing copy_from to smithi02683976-21 2024-09-13T14:19:52.910 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 22 (ObjNum 57 snap 4 seq_num 57) dirty exists 2024-09-13T14:19:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:52 smithi026 ceph-mon[39182]: pgmap v152: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:52 smithi026 ceph-mon[39182]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:19:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:52 smithi026 ceph-mon[27770]: pgmap v152: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:52 smithi026 ceph-mon[27770]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:19:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:52 smithi145 ceph-mon[31057]: pgmap v152: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-13T14:19:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:52 smithi145 ceph-mon[31057]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:19:53.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:19:52 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:19:53.726 INFO:tasks.rados.rados.0.smithi026.stdout:248: done (6 left) 2024-09-13T14:19:53.726 INFO:tasks.rados.rados.0.smithi026.stdout:249: done (5 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:250: done (4 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:252: done (3 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:254: done (2 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:255: done (1 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:256: done (0 left) 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 17 2024-09-13T14:19:53.727 INFO:tasks.rados.rados.0.smithi026.stdout:257: seq_num 103 ranges {827392=532480,1359872=737280,2097152=466944} 2024-09-13T14:19:53.733 INFO:tasks.rados.rados.0.smithi026.stdout:257: writing smithi02683976-16 from 827392 to 1359872 tid 1 2024-09-13T14:19:53.738 INFO:tasks.rados.rados.0.smithi026.stdout:257: writing smithi02683976-16 from 1359872 to 2097152 tid 2 2024-09-13T14:19:53.741 INFO:tasks.rados.rados.0.smithi026.stdout:257: writing smithi02683976-16 from 2097152 to 2564096 tid 3 2024-09-13T14:19:53.742 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 17 2024-09-13T14:19:53.742 INFO:tasks.rados.rados.0.smithi026.stdout:257: finishing write tid 1 to smithi02683976-16 2024-09-13T14:19:53.742 INFO:tasks.rados.rados.0.smithi026.stdout:258: seq_num 104 ranges {1499136=548864,2048000=622592,2670592=581632} 2024-09-13T14:19:53.753 INFO:tasks.rados.rados.0.smithi026.stdout:258: writing smithi02683976-18 from 1499136 to 2048000 tid 1 2024-09-13T14:19:53.757 INFO:tasks.rados.rados.0.smithi026.stdout:258: writing smithi02683976-18 from 2048000 to 2670592 tid 2 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:258: writing smithi02683976-18 from 2670592 to 3252224 tid 3 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:259: setattr oid 45 current snap is 17 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:257: finishing write tid 2 to smithi02683976-16 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:257: finishing write tid 3 to smithi02683976-16 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:257: finishing write tid 4 to smithi02683976-16 2024-09-13T14:19:53.761 INFO:tasks.rados.rados.0.smithi026.stdout:257: finishing write tid 5 to smithi02683976-16 2024-09-13T14:19:53.762 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 92 (ObjNum 103 snap 17 seq_num 103) dirty exists 2024-09-13T14:19:53.762 INFO:tasks.rados.rados.0.smithi026.stdout:257: left oid 16 (ObjNum 103 snap 17 seq_num 103) 2024-09-13T14:19:53.762 INFO:tasks.rados.rados.0.smithi026.stdout:258: finishing write tid 1 to smithi02683976-18 2024-09-13T14:19:53.762 INFO:tasks.rados.rados.0.smithi026.stdout:258: finishing write tid 2 to smithi02683976-18 2024-09-13T14:19:53.763 INFO:tasks.rados.rados.0.smithi026.stdout:258: finishing write tid 3 to smithi02683976-18 2024-09-13T14:19:53.763 INFO:tasks.rados.rados.0.smithi026.stdout:258: finishing write tid 4 to smithi02683976-18 2024-09-13T14:19:53.763 INFO:tasks.rados.rados.0.smithi026.stdout:257: done (2 left) 2024-09-13T14:19:53.763 INFO:tasks.rados.rados.0.smithi026.stdout:260: snap_remove snap 9 2024-09-13T14:19:54.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:53 smithi026 ceph-mon[27770]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:19:54.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:53 smithi026 ceph-mon[27770]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:19:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:53 smithi026 ceph-mon[39182]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:19:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:53 smithi026 ceph-mon[39182]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:19:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:53 smithi145 ceph-mon[31057]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:19:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:53 smithi145 ceph-mon[31057]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:19:54.728 INFO:tasks.rados.rados.0.smithi026.stdout:260: done (2 left) 2024-09-13T14:19:54.728 INFO:tasks.rados.rados.0.smithi026.stdout:261: setattr oid 15 current snap is 17 2024-09-13T14:19:54.729 INFO:tasks.rados.rados.0.smithi026.stdout:258: finishing write tid 5 to smithi02683976-18 2024-09-13T14:19:54.729 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 107 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:19:54.729 INFO:tasks.rados.rados.0.smithi026.stdout:258: left oid 18 (ObjNum 104 snap 17 seq_num 104) 2024-09-13T14:19:54.729 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 20 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-13T14:19:54.731 INFO:tasks.rados.rados.0.smithi026.stdout:258: done (2 left) 2024-09-13T14:19:54.732 INFO:tasks.rados.rados.0.smithi026.stdout:259: done (1 left) 2024-09-13T14:19:54.732 INFO:tasks.rados.rados.0.smithi026.stdout:262: snap_create 2024-09-13T14:19:54.738 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 72 (ObjNum 95 snap 16 seq_num 95) dirty exists 2024-09-13T14:19:55.025 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:54 smithi026 ceph-mon[39182]: pgmap v155: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:55.025 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:54 smithi026 ceph-mon[39182]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:19:55.026 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:54 smithi026 ceph-mon[27770]: pgmap v155: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:55.026 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:54 smithi026 ceph-mon[27770]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:19:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:54 smithi145 ceph-mon[31057]: pgmap v155: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:54 smithi145 ceph-mon[31057]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:19:55.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:19:55 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:19:55] "GET /metrics HTTP/1.1" 200 239806 "" "Prometheus/2.43.0" 2024-09-13T14:19:55.729 INFO:tasks.rados.rados.0.smithi026.stdout:261: done (1 left) 2024-09-13T14:19:55.729 INFO:tasks.rados.rados.0.smithi026.stdout:262: done (0 left) 2024-09-13T14:19:55.729 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 18 2024-09-13T14:19:55.729 INFO:tasks.rados.rados.0.smithi026.stdout:263: seq_num 105 ranges {540672=565248,1105920=507904,1613824=507904,2121728=188416} 2024-09-13T14:19:55.735 INFO:tasks.rados.rados.0.smithi026.stdout:263: writing smithi02683976-33 from 540672 to 1105920 tid 1 2024-09-13T14:19:55.738 INFO:tasks.rados.rados.0.smithi026.stdout:263: writing smithi02683976-33 from 1105920 to 1613824 tid 2 2024-09-13T14:19:55.741 INFO:tasks.rados.rados.0.smithi026.stdout:263: writing smithi02683976-33 from 1613824 to 2121728 tid 3 2024-09-13T14:19:55.743 INFO:tasks.rados.rados.0.smithi026.stdout:263: writing smithi02683976-33 from 2121728 to 2310144 tid 4 2024-09-13T14:19:55.743 INFO:tasks.rados.rados.0.smithi026.stdout:264: copy_from oid 18 from oid 13 current snap is 18 2024-09-13T14:19:55.743 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 1 to smithi02683976-33 2024-09-13T14:19:55.743 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 2 to smithi02683976-33 2024-09-13T14:19:55.743 INFO:tasks.rados.rados.0.smithi026.stdout:265: snap_create 2024-09-13T14:19:55.746 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 3 to smithi02683976-33 2024-09-13T14:19:55.746 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 4 to smithi02683976-33 2024-09-13T14:19:55.746 INFO:tasks.rados.rados.0.smithi026.stdout:264: finishing copy_from to smithi02683976-18 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:264: got expected ENOENT (src dne) 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 5 to smithi02683976-33 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:264: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:263: finishing write tid 6 to smithi02683976-33 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 128 (ObjNum 105 snap 18 seq_num 105) dirty exists 2024-09-13T14:19:55.747 INFO:tasks.rados.rados.0.smithi026.stdout:263: left oid 33 (ObjNum 105 snap 18 seq_num 105) 2024-09-13T14:19:56.730 INFO:tasks.rados.rados.0.smithi026.stdout:263: done (2 left) 2024-09-13T14:19:56.730 INFO:tasks.rados.rados.0.smithi026.stdout:264: done (1 left) 2024-09-13T14:19:56.730 INFO:tasks.rados.rados.0.smithi026.stdout:265: done (0 left) 2024-09-13T14:19:56.731 INFO:tasks.rados.rados.0.smithi026.stdout:266: snap_create 2024-09-13T14:19:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:56 smithi026 ceph-mon[27770]: pgmap v158: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:56 smithi026 ceph-mon[27770]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:19:56.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:56 smithi026 ceph-mon[39182]: pgmap v158: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:56 smithi026 ceph-mon[39182]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:19:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:56 smithi145 ceph-mon[31057]: pgmap v158: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2024-09-13T14:19:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:56 smithi145 ceph-mon[31057]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:19:57.741 INFO:tasks.rados.rados.0.smithi026.stdout:266: done (0 left) 2024-09-13T14:19:57.741 INFO:tasks.rados.rados.0.smithi026.stdout:267: read oid 15 snap -1 2024-09-13T14:19:57.741 INFO:tasks.rados.rados.0.smithi026.stdout:267: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-13T14:19:57.741 INFO:tasks.rados.rados.0.smithi026.stdout:268: snap_create 2024-09-13T14:19:57.757 INFO:tasks.rados.rados.0.smithi026.stdout:267: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-13T14:19:58.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:57 smithi026 ceph-mon[27770]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:19:58.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:57 smithi026 ceph-mon[39182]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:19:58.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:57 smithi145 ceph-mon[31057]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:19:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:267: done (1 left) 2024-09-13T14:19:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:268: done (0 left) 2024-09-13T14:19:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 21 2024-09-13T14:19:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:269: seq_num 106 ranges {761856=581632,1343488=794624,2138112=409600} 2024-09-13T14:19:58.751 INFO:tasks.rados.rados.0.smithi026.stdout:269: writing smithi02683976-23 from 761856 to 1343488 tid 1 2024-09-13T14:19:58.756 INFO:tasks.rados.rados.0.smithi026.stdout:269: writing smithi02683976-23 from 1343488 to 2138112 tid 2 2024-09-13T14:19:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:269: writing smithi02683976-23 from 2138112 to 2547712 tid 3 2024-09-13T14:19:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:270: rollback oid 9 current snap is 21 2024-09-13T14:19:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 19 2024-09-13T14:19:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:269: finishing write tid 1 to smithi02683976-23 2024-09-13T14:19:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:271: snap_remove snap 18 2024-09-13T14:19:59.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:58 smithi026 ceph-mon[27770]: pgmap v161: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:59.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:58 smithi026 ceph-mon[27770]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:19:59.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:58 smithi026 ceph-mon[39182]: pgmap v161: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:59.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:58 smithi026 ceph-mon[39182]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:19:59.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:58 smithi145 ceph-mon[31057]: pgmap v161: 121 pgs: 121 active+clean; 66 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:59.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:58 smithi145 ceph-mon[31057]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:271: done (2 left) 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:272: rollback oid 25 current snap is 21 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 20 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:269: finishing write tid 2 to smithi02683976-23 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:270: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:269: finishing write tid 3 to smithi02683976-23 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:269: finishing write tid 4 to smithi02683976-23 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:270: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:270: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:19:59.752 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 96 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:269: finishing write tid 5 to smithi02683976-23 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 116 (ObjNum 106 snap 21 seq_num 106) dirty exists 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:269: left oid 23 (ObjNum 106 snap 21 seq_num 106) 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:269: done (2 left) 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:270: done (1 left) 2024-09-13T14:19:59.753 INFO:tasks.rados.rados.0.smithi026.stdout:273: snap_create 2024-09-13T14:19:59.757 INFO:tasks.rados.rados.0.smithi026.stdout:272: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:19:59.757 INFO:tasks.rados.rados.0.smithi026.stdout:272: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:19:59.758 INFO:tasks.rados.rados.0.smithi026.stdout:272: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:19:59.758 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 104 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-13T14:20:00.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:19:59 smithi026 ceph-mon[27770]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:20:00.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:19:59 smithi026 ceph-mon[39182]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:20:00.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:19:59 smithi145 ceph-mon[31057]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:20:00.758 INFO:tasks.rados.rados.0.smithi026.stdout:272: done (1 left) 2024-09-13T14:20:00.758 INFO:tasks.rados.rados.0.smithi026.stdout:273: done (0 left) 2024-09-13T14:20:00.759 INFO:tasks.rados.rados.0.smithi026.stdout:274: read oid 23 snap -1 2024-09-13T14:20:00.759 INFO:tasks.rados.rados.0.smithi026.stdout:274: expect (ObjNum 106 snap 21 seq_num 106) 2024-09-13T14:20:00.759 INFO:tasks.rados.rados.0.smithi026.stdout:275: snap_remove snap 19 2024-09-13T14:20:01.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[27770]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:01.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[27770]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:20:01.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[27770]: overall HEALTH_OK 2024-09-13T14:20:01.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[39182]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:01.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[39182]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:20:01.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:00 smithi026 ceph-mon[39182]: overall HEALTH_OK 2024-09-13T14:20:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:00 smithi145 ceph-mon[31057]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:00 smithi145 ceph-mon[31057]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:20:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:00 smithi145 ceph-mon[31057]: overall HEALTH_OK 2024-09-13T14:20:01.762 INFO:tasks.rados.rados.0.smithi026.stdout:275: done (1 left) 2024-09-13T14:20:01.763 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 22 2024-09-13T14:20:01.763 INFO:tasks.rados.rados.0.smithi026.stdout:274: expect (ObjNum 106 snap 21 seq_num 106) 2024-09-13T14:20:01.843 INFO:tasks.rados.rados.0.smithi026.stdout:276: seq_num 107 ranges {745472=598016,1343488=729088,2072576=466944,2539520=8192} 2024-09-13T14:20:01.850 INFO:tasks.rados.rados.0.smithi026.stdout:276: writing smithi02683976-45 from 745472 to 1343488 tid 1 2024-09-13T14:20:01.854 INFO:tasks.rados.rados.0.smithi026.stdout:276: writing smithi02683976-45 from 1343488 to 2072576 tid 2 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:276: writing smithi02683976-45 from 2072576 to 2539520 tid 3 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:276: writing smithi02683976-45 from 2539520 to 2547712 tid 4 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:274: done (1 left) 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:277: rollback oid 48 current snap is 22 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 16 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 1 to smithi02683976-45 2024-09-13T14:20:01.858 INFO:tasks.rados.rados.0.smithi026.stdout:278: snap_create 2024-09-13T14:20:01.860 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 2 to smithi02683976-45 2024-09-13T14:20:01.861 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 3 to smithi02683976-45 2024-09-13T14:20:01.861 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 4 to smithi02683976-45 2024-09-13T14:20:01.862 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 5 to smithi02683976-45 2024-09-13T14:20:01.863 INFO:tasks.rados.rados.0.smithi026.stdout:277: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:20:01.863 INFO:tasks.rados.rados.0.smithi026.stdout:277: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:20:01.865 INFO:tasks.rados.rados.0.smithi026.stdout:277: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:20:01.865 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 30 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:20:01.866 INFO:tasks.rados.rados.0.smithi026.stdout:276: finishing write tid 6 to smithi02683976-45 2024-09-13T14:20:01.866 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 28 (ObjNum 107 snap 22 seq_num 107) dirty exists 2024-09-13T14:20:01.866 INFO:tasks.rados.rados.0.smithi026.stdout:276: left oid 45 (ObjNum 107 snap 22 seq_num 107) 2024-09-13T14:20:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:01 smithi026 ceph-mon[39182]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:20:02.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:01 smithi026 ceph-mon[27770]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:20:02.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:01 smithi145 ceph-mon[31057]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:20:02.764 INFO:tasks.rados.rados.0.smithi026.stdout:276: done (2 left) 2024-09-13T14:20:02.765 INFO:tasks.rados.rados.0.smithi026.stdout:277: done (1 left) 2024-09-13T14:20:02.765 INFO:tasks.rados.rados.0.smithi026.stdout:278: done (0 left) 2024-09-13T14:20:02.765 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 23 2024-09-13T14:20:02.765 INFO:tasks.rados.rados.0.smithi026.stdout:279: seq_num 108 ranges {3252224=614400,3866624=606208,4472832=598016} 2024-09-13T14:20:02.785 INFO:tasks.rados.rados.0.smithi026.stdout:279: writing smithi02683976-18 from 3252224 to 3866624 tid 1 2024-09-13T14:20:02.789 INFO:tasks.rados.rados.0.smithi026.stdout:279: writing smithi02683976-18 from 3866624 to 4472832 tid 2 2024-09-13T14:20:02.793 INFO:tasks.rados.rados.0.smithi026.stdout:279: writing smithi02683976-18 from 4472832 to 5070848 tid 3 2024-09-13T14:20:02.793 INFO:tasks.rados.rados.0.smithi026.stdout:280: read oid 31 snap -1 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:280: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:281: read oid 5 snap -1 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:281: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:282: copy_from oid 21 from oid 32 current snap is 23 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:279: finishing write tid 1 to smithi02683976-18 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:279: finishing write tid 2 to smithi02683976-18 2024-09-13T14:20:02.794 INFO:tasks.rados.rados.0.smithi026.stdout:283: snap_create 2024-09-13T14:20:02.796 INFO:tasks.rados.rados.0.smithi026.stdout:279: finishing write tid 3 to smithi02683976-18 2024-09-13T14:20:02.796 INFO:tasks.rados.rados.0.smithi026.stdout:281: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-13T14:20:02.799 INFO:tasks.rados.rados.0.smithi026.stdout:279: finishing write tid 4 to smithi02683976-18 2024-09-13T14:20:02.799 INFO:tasks.rados.rados.0.smithi026.stdout:279: finishing write tid 5 to smithi02683976-18 2024-09-13T14:20:02.799 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 125 (ObjNum 108 snap 23 seq_num 108) dirty exists 2024-09-13T14:20:02.799 INFO:tasks.rados.rados.0.smithi026.stdout:279: left oid 18 (ObjNum 108 snap 23 seq_num 108) 2024-09-13T14:20:02.799 INFO:tasks.rados.rados.0.smithi026.stdout:280: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-13T14:20:02.813 INFO:tasks.rados.rados.0.smithi026.stdout:282: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:20:02.813 INFO:tasks.rados.rados.0.smithi026.stdout:282: finishing copy_from to smithi02683976-21 2024-09-13T14:20:02.813 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:03.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[39182]: pgmap v167: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:03.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[39182]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:20:03.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[27770]: pgmap v167: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[27770]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:20:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:02 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:03.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:20:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:02 smithi145 ceph-mon[31057]: pgmap v167: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-13T14:20:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:02 smithi145 ceph-mon[31057]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:20:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:02 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:279: done (4 left) 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:280: done (3 left) 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:281: done (2 left) 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:282: done (1 left) 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:283: done (0 left) 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:284: read oid 41 snap -1 2024-09-13T14:20:03.729 INFO:tasks.rados.rados.0.smithi026.stdout:284: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-13T14:20:03.730 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 24 2024-09-13T14:20:03.730 INFO:tasks.rados.rados.0.smithi026.stdout:285: seq_num 109 ranges {2564096=630784,3194880=491520,3686400=483328,4169728=229376} 2024-09-13T14:20:03.746 INFO:tasks.rados.rados.0.smithi026.stdout:285: writing smithi02683976-16 from 2564096 to 3194880 tid 1 2024-09-13T14:20:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:285: writing smithi02683976-16 from 3194880 to 3686400 tid 2 2024-09-13T14:20:03.753 INFO:tasks.rados.rados.0.smithi026.stdout:285: writing smithi02683976-16 from 3686400 to 4169728 tid 3 2024-09-13T14:20:03.755 INFO:tasks.rados.rados.0.smithi026.stdout:285: writing smithi02683976-16 from 4169728 to 4399104 tid 4 2024-09-13T14:20:03.755 INFO:tasks.rados.rados.0.smithi026.stdout:286: setattr oid 13 current snap is 24 2024-09-13T14:20:03.755 INFO:tasks.rados.rados.0.smithi026.stdout:284: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-13T14:20:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 1 to smithi02683976-16 2024-09-13T14:20:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 2 to smithi02683976-16 2024-09-13T14:20:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 3 to smithi02683976-16 2024-09-13T14:20:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 4 to smithi02683976-16 2024-09-13T14:20:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 5 to smithi02683976-16 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:285: finishing write tid 6 to smithi02683976-16 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 108 (ObjNum 109 snap 24 seq_num 109) dirty exists 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:285: left oid 16 (ObjNum 109 snap 24 seq_num 109) 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:284: done (2 left) 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:285: done (1 left) 2024-09-13T14:20:03.806 INFO:tasks.rados.rados.0.smithi026.stdout:287: delete oid 31 current snap is 24 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:287: done (1 left) 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:288: read oid 21 snap -1 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:288: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:289: read oid 28 snap 22 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:289: expect (ObjNum 93 snap 15 seq_num 93) 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:290: delete oid 8 current snap is 24 2024-09-13T14:20:03.807 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 109 (ObjNum 113 snap 0 seq_num 3890191680) dirty exists 2024-09-13T14:20:03.808 INFO:tasks.rados.rados.0.smithi026.stdout:286: done (3 left) 2024-09-13T14:20:03.809 INFO:tasks.rados.rados.0.smithi026.stdout:290: done (2 left) 2024-09-13T14:20:03.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 24 2024-09-13T14:20:03.809 INFO:tasks.rados.rados.0.smithi026.stdout:291: seq_num 112 ranges {2629632=679936,3309568=548864,3858432=655360} 2024-09-13T14:20:03.827 INFO:tasks.rados.rados.0.smithi026.stdout:291: writing smithi02683976-15 from 2629632 to 3309568 tid 1 2024-09-13T14:20:03.830 INFO:tasks.rados.rados.0.smithi026.stdout:291: writing smithi02683976-15 from 3309568 to 3858432 tid 2 2024-09-13T14:20:03.834 INFO:tasks.rados.rados.0.smithi026.stdout:291: writing smithi02683976-15 from 3858432 to 4513792 tid 3 2024-09-13T14:20:03.834 INFO:tasks.rados.rados.0.smithi026.stdout:292: delete oid 26 current snap is 24 2024-09-13T14:20:03.834 INFO:tasks.rados.rados.0.smithi026.stdout:288: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:03.837 INFO:tasks.rados.rados.0.smithi026.stdout:289: expect (ObjNum 93 snap 15 seq_num 93) 2024-09-13T14:20:03.898 INFO:tasks.rados.rados.0.smithi026.stdout:291: finishing write tid 1 to smithi02683976-15 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: finishing write tid 2 to smithi02683976-15 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: finishing write tid 3 to smithi02683976-15 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: finishing write tid 4 to smithi02683976-15 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: finishing write tid 5 to smithi02683976-15 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 79 (ObjNum 112 snap 24 seq_num 112) dirty exists 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: left oid 15 (ObjNum 112 snap 24 seq_num 112) 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:288: done (3 left) 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:289: done (2 left) 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:291: done (1 left) 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:292: done (0 left) 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:293: copy_from oid 32 from oid 23 current snap is 24 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:294: read oid 7 snap -1 2024-09-13T14:20:03.899 INFO:tasks.rados.rados.0.smithi026.stdout:294: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:295: read oid 19 snap -1 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:295: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:296: copy_from oid 5 from oid 44 current snap is 24 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:297: rmattr oid 36 current snap is 24 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:297: done (4 left) 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:298: read oid 45 snap -1 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:298: expect (ObjNum 107 snap 22 seq_num 107) 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:299: delete oid 8 current snap is 24 2024-09-13T14:20:03.900 INFO:tasks.rados.rados.0.smithi026.stdout:299: done (5 left) 2024-09-13T14:20:03.901 INFO:tasks.rados.rados.0.smithi026.stdout:300: delete oid 48 current snap is 24 2024-09-13T14:20:03.901 INFO:tasks.rados.rados.0.smithi026.stdout:294: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:03.904 INFO:tasks.rados.rados.0.smithi026.stdout:295: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-13T14:20:03.914 INFO:tasks.rados.rados.0.smithi026.stdout:293: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:20:03.914 INFO:tasks.rados.rados.0.smithi026.stdout:293: finishing copy_from to smithi02683976-32 2024-09-13T14:20:03.915 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 116 (ObjNum 106 snap 21 seq_num 106) dirty exists 2024-09-13T14:20:03.915 INFO:tasks.rados.rados.0.smithi026.stdout:298: expect (ObjNum 107 snap 22 seq_num 107) 2024-09-13T14:20:04.000 INFO:tasks.rados.rados.0.smithi026.stdout:296: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:20:04.000 INFO:tasks.rados.rados.0.smithi026.stdout:296: finishing copy_from to smithi02683976-5 2024-09-13T14:20:04.000 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 85 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:293: done (5 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:294: done (4 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:295: done (3 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:296: done (2 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:298: done (1 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:300: done (0 left) 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 24 2024-09-13T14:20:04.001 INFO:tasks.rados.rados.0.smithi026.stdout:301: seq_num 116 ranges {1703936=745472,2449408=532480,2981888=638976,3620864=32768} 2024-09-13T14:20:04.013 INFO:tasks.rados.rados.0.smithi026.stdout:301: writing smithi02683976-42 from 1703936 to 2449408 tid 1 2024-09-13T14:20:04.016 INFO:tasks.rados.rados.0.smithi026.stdout:301: writing smithi02683976-42 from 2449408 to 2981888 tid 2 2024-09-13T14:20:04.020 INFO:tasks.rados.rados.0.smithi026.stdout:301: writing smithi02683976-42 from 2981888 to 3620864 tid 3 2024-09-13T14:20:04.020 INFO:tasks.rados.rados.0.smithi026.stdout:301: writing smithi02683976-42 from 3620864 to 3653632 tid 4 2024-09-13T14:20:04.020 INFO:tasks.rados.rados.0.smithi026.stdout:302: snap_create 2024-09-13T14:20:04.021 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 1 to smithi02683976-42 2024-09-13T14:20:04.021 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 2 to smithi02683976-42 2024-09-13T14:20:04.023 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 3 to smithi02683976-42 2024-09-13T14:20:04.023 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 4 to smithi02683976-42 2024-09-13T14:20:04.023 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 5 to smithi02683976-42 2024-09-13T14:20:04.024 INFO:tasks.rados.rados.0.smithi026.stdout:301: finishing write tid 6 to smithi02683976-42 2024-09-13T14:20:04.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 54 (ObjNum 116 snap 24 seq_num 116) dirty exists 2024-09-13T14:20:04.024 INFO:tasks.rados.rados.0.smithi026.stdout:301: left oid 42 (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:20:04.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:03 smithi026 ceph-mon[39182]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:20:04.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:03 smithi026 ceph-mon[39182]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:20:04.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:03 smithi026 ceph-mon[27770]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:20:04.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:03 smithi026 ceph-mon[27770]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:20:04.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:03 smithi145 ceph-mon[31057]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:20:04.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:03 smithi145 ceph-mon[31057]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:20:04.779 INFO:tasks.rados.rados.0.smithi026.stdout:301: done (1 left) 2024-09-13T14:20:04.779 INFO:tasks.rados.rados.0.smithi026.stdout:302: done (0 left) 2024-09-13T14:20:04.779 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 25 2024-09-13T14:20:04.779 INFO:tasks.rados.rados.0.smithi026.stdout:303: seq_num 117 ranges {2064384=770048,2834432=417792,3252224=442368,3694592=344064} 2024-09-13T14:20:04.794 INFO:tasks.rados.rados.0.smithi026.stdout:303: writing smithi02683976-5 from 2064384 to 2834432 tid 1 2024-09-13T14:20:04.796 INFO:tasks.rados.rados.0.smithi026.stdout:303: writing smithi02683976-5 from 2834432 to 3252224 tid 2 2024-09-13T14:20:04.799 INFO:tasks.rados.rados.0.smithi026.stdout:303: writing smithi02683976-5 from 3252224 to 3694592 tid 3 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:303: writing smithi02683976-5 from 3694592 to 4038656 tid 4 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:304: rollback oid 31 current snap is 25 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 23 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 1 to smithi02683976-5 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 2 to smithi02683976-5 2024-09-13T14:20:04.802 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 25 2024-09-13T14:20:04.803 INFO:tasks.rados.rados.0.smithi026.stdout:305: seq_num 118 ranges {3063808=786432,3850240=704512,4554752=499712} 2024-09-13T14:20:04.823 INFO:tasks.rados.rados.0.smithi026.stdout:305: writing smithi02683976-49 from 3063808 to 3850240 tid 1 2024-09-13T14:20:04.827 INFO:tasks.rados.rados.0.smithi026.stdout:305: writing smithi02683976-49 from 3850240 to 4554752 tid 2 2024-09-13T14:20:04.831 INFO:tasks.rados.rados.0.smithi026.stdout:305: writing smithi02683976-49 from 4554752 to 5054464 tid 3 2024-09-13T14:20:04.831 INFO:tasks.rados.rados.0.smithi026.stdout:306: rmattr oid 41 current snap is 25 2024-09-13T14:20:04.831 INFO:tasks.rados.rados.0.smithi026.stdout:306: done (3 left) 2024-09-13T14:20:04.831 INFO:tasks.rados.rados.0.smithi026.stdout:307: read oid 7 snap -1 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:307: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:308: snap_create 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 3 to smithi02683976-5 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:304: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:304: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 106 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 4 to smithi02683976-5 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 5 to smithi02683976-5 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:303: finishing write tid 6 to smithi02683976-5 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 136 (ObjNum 117 snap 25 seq_num 117) dirty exists 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:303: left oid 5 (ObjNum 117 snap 25 seq_num 117) 2024-09-13T14:20:04.832 INFO:tasks.rados.rados.0.smithi026.stdout:305: finishing write tid 1 to smithi02683976-49 2024-09-13T14:20:04.833 INFO:tasks.rados.rados.0.smithi026.stdout:305: finishing write tid 2 to smithi02683976-49 2024-09-13T14:20:04.833 INFO:tasks.rados.rados.0.smithi026.stdout:305: finishing write tid 3 to smithi02683976-49 2024-09-13T14:20:04.833 INFO:tasks.rados.rados.0.smithi026.stdout:305: finishing write tid 4 to smithi02683976-49 2024-09-13T14:20:04.834 INFO:tasks.rados.rados.0.smithi026.stdout:307: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:04.836 INFO:tasks.rados.rados.0.smithi026.stdout:305: finishing write tid 5 to smithi02683976-49 2024-09-13T14:20:04.837 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 82 (ObjNum 118 snap 25 seq_num 118) dirty exists 2024-09-13T14:20:04.837 INFO:tasks.rados.rados.0.smithi026.stdout:305: left oid 49 (ObjNum 118 snap 25 seq_num 118) 2024-09-13T14:20:05.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:04 smithi026 ceph-mon[39182]: pgmap v170: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:05.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:04 smithi026 ceph-mon[27770]: pgmap v170: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:05.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:04 smithi145 ceph-mon[31057]: pgmap v170: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:05.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:303: done (4 left) 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:304: done (3 left) 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:305: done (2 left) 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:307: done (1 left) 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:308: done (0 left) 2024-09-13T14:20:05.785 INFO:tasks.rados.rados.0.smithi026.stdout:309: delete oid 49 current snap is 26 2024-09-13T14:20:05.788 INFO:tasks.rados.rados.0.smithi026.stdout:309: done (0 left) 2024-09-13T14:20:05.789 INFO:tasks.rados.rados.0.smithi026.stdout:310: rollback oid 18 current snap is 26 2024-09-13T14:20:05.789 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 24 2024-09-13T14:20:05.789 INFO:tasks.rados.rados.0.smithi026.stdout:311: snap_create 2024-09-13T14:20:05.792 INFO:tasks.rados.rados.0.smithi026.stdout:310: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:20:05.792 INFO:tasks.rados.rados.0.smithi026.stdout:310: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:20:05.792 INFO:tasks.rados.rados.0.smithi026.stdout:310: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:20:05.792 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 131 (ObjNum 108 snap 23 seq_num 108) dirty exists 2024-09-13T14:20:06.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:05 smithi026 ceph-mon[39182]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:20:06.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:05 smithi026 ceph-mon[27770]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:20:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:05 smithi145 ceph-mon[31057]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:20:06.789 INFO:tasks.rados.rados.0.smithi026.stdout:310: done (1 left) 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:311: done (0 left) 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:312: rollback oid 41 current snap is 27 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 41 to 25 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:313: rollback oid 18 current snap is 27 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 20 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:314: rmattr oid 23 current snap is 27 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:314: done (2 left) 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:315: copy_from oid 43 from oid 10 current snap is 27 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 27 2024-09-13T14:20:06.790 INFO:tasks.rados.rados.0.smithi026.stdout:316: seq_num 120 ranges {303104=417792,720896=475136,1196032=622592,1818624=507904} 2024-09-13T14:20:06.794 INFO:tasks.rados.rados.0.smithi026.stdout:316: writing smithi02683976-19 from 303104 to 720896 tid 1 2024-09-13T14:20:06.797 INFO:tasks.rados.rados.0.smithi026.stdout:316: writing smithi02683976-19 from 720896 to 1196032 tid 2 2024-09-13T14:20:06.801 INFO:tasks.rados.rados.0.smithi026.stdout:316: writing smithi02683976-19 from 1196032 to 1818624 tid 3 2024-09-13T14:20:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:316: writing smithi02683976-19 from 1818624 to 2326528 tid 4 2024-09-13T14:20:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 27 2024-09-13T14:20:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:312: finishing rollback tid 0 to smithi02683976-41 2024-09-13T14:20:06.805 INFO:tasks.rados.rados.0.smithi026.stdout:312: finishing rollback tid 1 to smithi02683976-41 2024-09-13T14:20:06.805 INFO:tasks.rados.rados.0.smithi026.stdout:312: finishing rollback tid 2 to smithi02683976-41 2024-09-13T14:20:06.805 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 84 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-13T14:20:06.805 INFO:tasks.rados.rados.0.smithi026.stdout:313: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:20:06.805 INFO:tasks.rados.rados.0.smithi026.stdout:317: seq_num 121 ranges {0=434176,434176=753664,1187840=425984,1613824=425984} 2024-09-13T14:20:06.806 INFO:tasks.rados.rados.0.smithi026.stdout:317: writing smithi02683976-13 from 0 to 434176 tid 1 2024-09-13T14:20:06.811 INFO:tasks.rados.rados.0.smithi026.stdout:317: writing smithi02683976-13 from 434176 to 1187840 tid 2 2024-09-13T14:20:06.813 INFO:tasks.rados.rados.0.smithi026.stdout:317: writing smithi02683976-13 from 1187840 to 1613824 tid 3 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:317: writing smithi02683976-13 from 1613824 to 2039808 tid 4 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:312: done (4 left) 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:318: setattr oid 41 current snap is 27 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:313: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:313: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 135 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:315: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 1 to smithi02683976-19 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 2 to smithi02683976-19 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:315: finishing copy_from to smithi02683976-43 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 43 (ObjNum 78 snap 10 seq_num 78) dirty exists 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 3 to smithi02683976-19 2024-09-13T14:20:06.817 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 4 to smithi02683976-19 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 5 to smithi02683976-19 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:316: finishing write tid 6 to smithi02683976-19 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 117 (ObjNum 120 snap 27 seq_num 120) dirty exists 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:316: left oid 19 (ObjNum 120 snap 27 seq_num 120) 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 1 to smithi02683976-13 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 2 to smithi02683976-13 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 3 to smithi02683976-13 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:313: done (4 left) 2024-09-13T14:20:06.818 INFO:tasks.rados.rados.0.smithi026.stdout:315: done (3 left) 2024-09-13T14:20:06.819 INFO:tasks.rados.rados.0.smithi026.stdout:316: done (2 left) 2024-09-13T14:20:06.819 INFO:tasks.rados.rados.0.smithi026.stdout:319: snap_create 2024-09-13T14:20:06.819 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 4 to smithi02683976-13 2024-09-13T14:20:06.819 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 5 to smithi02683976-13 2024-09-13T14:20:06.820 INFO:tasks.rados.rados.0.smithi026.stdout:317: finishing write tid 6 to smithi02683976-13 2024-09-13T14:20:06.820 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 123 (ObjNum 121 snap 27 seq_num 121) dirty exists 2024-09-13T14:20:06.820 INFO:tasks.rados.rados.0.smithi026.stdout:317: left oid 13 (ObjNum 121 snap 27 seq_num 121) 2024-09-13T14:20:06.820 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 85 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-13T14:20:07.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:06 smithi026 ceph-mon[39182]: pgmap v173: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:07.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:06 smithi026 ceph-mon[39182]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:20:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:06 smithi026 ceph-mon[27770]: pgmap v173: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:06 smithi026 ceph-mon[27770]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:20:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:06 smithi145 ceph-mon[31057]: pgmap v173: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:20:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:06 smithi145 ceph-mon[31057]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:20:07.794 INFO:tasks.rados.rados.0.smithi026.stdout:317: done (2 left) 2024-09-13T14:20:07.794 INFO:tasks.rados.rados.0.smithi026.stdout:318: done (1 left) 2024-09-13T14:20:07.794 INFO:tasks.rados.rados.0.smithi026.stdout:319: done (0 left) 2024-09-13T14:20:07.794 INFO:tasks.rados.rados.0.smithi026.stdout:320: copy_from oid 34 from oid 46 current snap is 28 2024-09-13T14:20:07.794 INFO:tasks.rados.rados.0.smithi026.stdout:321: snap_remove snap 27 2024-09-13T14:20:08.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:07 smithi026 ceph-mon[39182]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:20:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:07 smithi026 ceph-mon[27770]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:20:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:07 smithi145 ceph-mon[31057]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:20:08.797 INFO:tasks.rados.rados.0.smithi026.stdout:320: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:20:08.797 INFO:tasks.rados.rados.0.smithi026.stdout:320: finishing copy_from to smithi02683976-34 2024-09-13T14:20:08.797 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 81 (ObjNum 87 snap 12 seq_num 87) dirty exists 2024-09-13T14:20:08.797 INFO:tasks.rados.rados.0.smithi026.stdout:320: done (1 left) 2024-09-13T14:20:08.797 INFO:tasks.rados.rados.0.smithi026.stdout:321: done (0 left) 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:322: rollback oid 46 current snap is 28 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 20 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:323: read oid 28 snap -1 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:323: expect (ObjNum 93 snap 15 seq_num 93) 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:324: read oid 36 snap -1 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:324: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-13T14:20:08.798 INFO:tasks.rados.rados.0.smithi026.stdout:325: delete oid 12 current snap is 28 2024-09-13T14:20:08.802 INFO:tasks.rados.rados.0.smithi026.stdout:325: done (3 left) 2024-09-13T14:20:08.802 INFO:tasks.rados.rados.0.smithi026.stdout:326: rollback oid 45 current snap is 28 2024-09-13T14:20:08.802 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 26 2024-09-13T14:20:08.802 INFO:tasks.rados.rados.0.smithi026.stdout:327: snap_remove snap 16 2024-09-13T14:20:09.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:08 smithi026 ceph-mon[27770]: pgmap v176: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:08 smithi026 ceph-mon[27770]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:20:09.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:08 smithi026 ceph-mon[39182]: pgmap v176: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:08 smithi026 ceph-mon[39182]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:20:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:08 smithi145 ceph-mon[31057]: pgmap v176: 121 pgs: 121 active+clean; 71 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:08 smithi145 ceph-mon[31057]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:20:09.808 INFO:tasks.rados.rados.0.smithi026.stdout:327: done (4 left) 2024-09-13T14:20:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 28 2024-09-13T14:20:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:328: seq_num 123 ranges {507904=466944,974848=573440,1548288=794624,2342912=237568} 2024-09-13T14:20:09.815 INFO:tasks.rados.rados.0.smithi026.stdout:328: writing smithi02683976-31 from 507904 to 974848 tid 1 2024-09-13T14:20:09.819 INFO:tasks.rados.rados.0.smithi026.stdout:328: writing smithi02683976-31 from 974848 to 1548288 tid 2 2024-09-13T14:20:09.824 INFO:tasks.rados.rados.0.smithi026.stdout:328: writing smithi02683976-31 from 1548288 to 2342912 tid 3 2024-09-13T14:20:09.825 INFO:tasks.rados.rados.0.smithi026.stdout:328: writing smithi02683976-31 from 2342912 to 2580480 tid 4 2024-09-13T14:20:09.825 INFO:tasks.rados.rados.0.smithi026.stdout:329: read oid 26 snap -1 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:329: expect deleted 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:330: setattr oid 3 current snap is 28 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:322: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:322: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:322: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 88 (ObjNum 87 snap 12 seq_num 87) dirty exists 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:326: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:326: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:326: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 32 (ObjNum 107 snap 22 seq_num 107) dirty exists 2024-09-13T14:20:09.826 INFO:tasks.rados.rados.0.smithi026.stdout:324: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-13T14:20:09.869 INFO:tasks.rados.rados.0.smithi026.stdout:323: expect (ObjNum 93 snap 15 seq_num 93) 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 1 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 2 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 3 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 4 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 5 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: finishing write tid 6 to smithi02683976-31 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 118 (ObjNum 123 snap 28 seq_num 123) dirty exists 2024-09-13T14:20:09.930 INFO:tasks.rados.rados.0.smithi026.stdout:328: left oid 31 (ObjNum 123 snap 28 seq_num 123) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:322: done (6 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:323: done (5 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:324: done (4 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:326: done (3 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:328: done (2 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:329: done (1 left) 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 28 2024-09-13T14:20:09.931 INFO:tasks.rados.rados.0.smithi026.stdout:331: seq_num 124 ranges {1654784=483328,2138112=458752,2596864=598016,3194880=548864} 2024-09-13T14:20:09.942 INFO:tasks.rados.rados.0.smithi026.stdout:331: writing smithi02683976-36 from 1654784 to 2138112 tid 1 2024-09-13T14:20:09.944 INFO:tasks.rados.rados.0.smithi026.stdout:331: writing smithi02683976-36 from 2138112 to 2596864 tid 2 2024-09-13T14:20:09.948 INFO:tasks.rados.rados.0.smithi026.stdout:331: writing smithi02683976-36 from 2596864 to 3194880 tid 3 2024-09-13T14:20:09.951 INFO:tasks.rados.rados.0.smithi026.stdout:331: writing smithi02683976-36 from 3194880 to 3743744 tid 4 2024-09-13T14:20:09.951 INFO:tasks.rados.rados.0.smithi026.stdout:332: snap_create 2024-09-13T14:20:09.952 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 138 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:20:09.952 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 1 to smithi02683976-36 2024-09-13T14:20:09.952 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 2 to smithi02683976-36 2024-09-13T14:20:09.952 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 3 to smithi02683976-36 2024-09-13T14:20:09.953 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 4 to smithi02683976-36 2024-09-13T14:20:09.953 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 5 to smithi02683976-36 2024-09-13T14:20:09.954 INFO:tasks.rados.rados.0.smithi026.stdout:331: finishing write tid 6 to smithi02683976-36 2024-09-13T14:20:09.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 148 (ObjNum 124 snap 28 seq_num 124) dirty exists 2024-09-13T14:20:09.954 INFO:tasks.rados.rados.0.smithi026.stdout:331: left oid 36 (ObjNum 124 snap 28 seq_num 124) 2024-09-13T14:20:10.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:09 smithi026 ceph-mon[27770]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:20:10.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:09 smithi026 ceph-mon[39182]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:20:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:09 smithi145 ceph-mon[31057]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:330: done (2 left) 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:331: done (1 left) 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:332: done (0 left) 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:333: read oid 25 snap 20 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:333: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-13T14:20:10.814 INFO:tasks.rados.rados.0.smithi026.stdout:334: snap_remove snap 26 2024-09-13T14:20:11.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:10 smithi026 ceph-mon[27770]: pgmap v179: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:11.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:10 smithi026 ceph-mon[27770]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:20:11.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:10 smithi026 ceph-mon[39182]: pgmap v179: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:11.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:10 smithi026 ceph-mon[39182]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:20:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:10 smithi145 ceph-mon[31057]: pgmap v179: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:10 smithi145 ceph-mon[31057]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:20:11.819 INFO:tasks.rados.rados.0.smithi026.stdout:334: done (1 left) 2024-09-13T14:20:11.819 INFO:tasks.rados.rados.0.smithi026.stdout:335: delete oid 36 current snap is 29 2024-09-13T14:20:11.819 INFO:tasks.rados.rados.0.smithi026.stdout:333: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-13T14:20:11.825 INFO:tasks.rados.rados.0.smithi026.stdout:333: done (1 left) 2024-09-13T14:20:11.825 INFO:tasks.rados.rados.0.smithi026.stdout:335: done (0 left) 2024-09-13T14:20:11.825 INFO:tasks.rados.rados.0.smithi026.stdout:336: snap_remove snap 17 2024-09-13T14:20:12.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:11 smithi026 ceph-mon[27770]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:20:12.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:11 smithi026 ceph-mon[27770]: pgmap v182: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:12.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:11 smithi026 ceph-mon[39182]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:20:12.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:11 smithi026 ceph-mon[39182]: pgmap v182: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:11 smithi145 ceph-mon[31057]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:20:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:11 smithi145 ceph-mon[31057]: pgmap v182: 121 pgs: 121 active+clean; 91 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.1 MiB/s wr, 32 op/s 2024-09-13T14:20:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:336: done (0 left) 2024-09-13T14:20:12.823 INFO:tasks.rados.rados.0.smithi026.stdout:337: snap_remove snap 28 2024-09-13T14:20:13.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:12 smithi026 ceph-mon[27770]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:20:13.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:12 smithi026 ceph-mon[39182]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:20:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:12 smithi145 ceph-mon[31057]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:20:13.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:20:13.732 INFO:tasks.rados.rados.0.smithi026.stdout:337: done (0 left) 2024-09-13T14:20:13.732 INFO:tasks.rados.rados.0.smithi026.stdout:338: copy_from oid 43 from oid 3 current snap is 29 2024-09-13T14:20:13.732 INFO:tasks.rados.rados.0.smithi026.stdout:339: read oid 37 snap -1 2024-09-13T14:20:13.733 INFO:tasks.rados.rados.0.smithi026.stdout:339: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-13T14:20:13.733 INFO:tasks.rados.rados.0.smithi026.stdout:340: delete oid 12 current snap is 29 2024-09-13T14:20:13.735 INFO:tasks.rados.rados.0.smithi026.stdout:340: done (2 left) 2024-09-13T14:20:13.735 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 29 2024-09-13T14:20:13.735 INFO:tasks.rados.rados.0.smithi026.stdout:341: seq_num 127 ranges {2039808=532480,2572288=507904,3080192=778240,3858432=319488} 2024-09-13T14:20:13.749 INFO:tasks.rados.rados.0.smithi026.stdout:341: writing smithi02683976-13 from 2039808 to 2572288 tid 1 2024-09-13T14:20:13.752 INFO:tasks.rados.rados.0.smithi026.stdout:341: writing smithi02683976-13 from 2572288 to 3080192 tid 2 2024-09-13T14:20:13.757 INFO:tasks.rados.rados.0.smithi026.stdout:341: writing smithi02683976-13 from 3080192 to 3858432 tid 3 2024-09-13T14:20:13.759 INFO:tasks.rados.rados.0.smithi026.stdout:341: writing smithi02683976-13 from 3858432 to 4177920 tid 4 2024-09-13T14:20:13.759 INFO:tasks.rados.rados.0.smithi026.stdout:342: delete oid 9 current snap is 29 2024-09-13T14:20:13.759 INFO:tasks.rados.rados.0.smithi026.stdout:339: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-13T14:20:13.760 INFO:tasks.rados.rados.0.smithi026.stdout:338: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:20:13.760 INFO:tasks.rados.rados.0.smithi026.stdout:338: finishing copy_from to smithi02683976-43 2024-09-13T14:20:13.760 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 138 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:20:13.760 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 1 to smithi02683976-13 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 2 to smithi02683976-13 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:338: done (3 left) 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:339: done (2 left) 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:342: done (1 left) 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 29 2024-09-13T14:20:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 3 to smithi02683976-13 2024-09-13T14:20:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:343: seq_num 129 ranges {40960=565248,606208=679936,1286144=786432,2072576=139264} 2024-09-13T14:20:13.764 INFO:tasks.rados.rados.0.smithi026.stdout:343: writing smithi02683976-37 from 40960 to 606208 tid 1 2024-09-13T14:20:13.768 INFO:tasks.rados.rados.0.smithi026.stdout:343: writing smithi02683976-37 from 606208 to 1286144 tid 2 2024-09-13T14:20:13.773 INFO:tasks.rados.rados.0.smithi026.stdout:343: writing smithi02683976-37 from 1286144 to 2072576 tid 3 2024-09-13T14:20:13.774 INFO:tasks.rados.rados.0.smithi026.stdout:343: writing smithi02683976-37 from 2072576 to 2211840 tid 4 2024-09-13T14:20:13.774 INFO:tasks.rados.rados.0.smithi026.stdout:344: copy_from oid 22 from oid 21 current snap is 29 2024-09-13T14:20:13.774 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 4 to smithi02683976-13 2024-09-13T14:20:13.774 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 5 to smithi02683976-13 2024-09-13T14:20:13.774 INFO:tasks.rados.rados.0.smithi026.stdout:341: finishing write tid 6 to smithi02683976-13 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 147 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:341: left oid 13 (ObjNum 127 snap 29 seq_num 127) 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 1 to smithi02683976-37 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 2 to smithi02683976-37 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:341: done (2 left) 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:345: setattr oid 43 current snap is 29 2024-09-13T14:20:13.775 INFO:tasks.rados.rados.0.smithi026.stdout:346: read oid 34 snap -1 2024-09-13T14:20:13.776 INFO:tasks.rados.rados.0.smithi026.stdout:346: expect (ObjNum 87 snap 12 seq_num 87) 2024-09-13T14:20:13.776 INFO:tasks.rados.rados.0.smithi026.stdout:347: snap_create 2024-09-13T14:20:13.776 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 3 to smithi02683976-37 2024-09-13T14:20:13.777 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 4 to smithi02683976-37 2024-09-13T14:20:13.777 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 5 to smithi02683976-37 2024-09-13T14:20:13.777 INFO:tasks.rados.rados.0.smithi026.stdout:344: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:20:13.777 INFO:tasks.rados.rados.0.smithi026.stdout:344: finishing copy_from to smithi02683976-22 2024-09-13T14:20:13.777 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:13.779 INFO:tasks.rados.rados.0.smithi026.stdout:343: finishing write tid 6 to smithi02683976-37 2024-09-13T14:20:13.779 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 64 (ObjNum 129 snap 29 seq_num 129) dirty exists 2024-09-13T14:20:13.779 INFO:tasks.rados.rados.0.smithi026.stdout:343: left oid 37 (ObjNum 129 snap 29 seq_num 129) 2024-09-13T14:20:13.780 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 148 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:20:13.788 INFO:tasks.rados.rados.0.smithi026.stdout:346: expect (ObjNum 87 snap 12 seq_num 87) 2024-09-13T14:20:14.084 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[39182]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:20:14.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[39182]: pgmap v185: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:14.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[39182]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:20:14.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[27770]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:20:14.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[27770]: pgmap v185: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:14.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:13 smithi026 ceph-mon[27770]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:20:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:13 smithi145 ceph-mon[31057]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:20:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:13 smithi145 ceph-mon[31057]: pgmap v185: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:13 smithi145 ceph-mon[31057]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:20:14.732 INFO:tasks.rados.rados.0.smithi026.stdout:343: done (4 left) 2024-09-13T14:20:14.732 INFO:tasks.rados.rados.0.smithi026.stdout:344: done (3 left) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:345: done (2 left) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:346: done (1 left) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:347: done (0 left) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:348: read oid 31 snap -1 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:348: expect (ObjNum 123 snap 28 seq_num 123) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:349: rollback oid 22 current snap is 30 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 25 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:350: read oid 48 snap -1 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:350: expect deleted 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:351: read oid 3 snap 23 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:351: expect (ObjNum 79 snap 10 seq_num 79) 2024-09-13T14:20:14.733 INFO:tasks.rados.rados.0.smithi026.stdout:352: snap_remove snap 21 2024-09-13T14:20:15.365 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:15] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2024-09-13T14:20:15.737 INFO:tasks.rados.rados.0.smithi026.stdout:352: done (4 left) 2024-09-13T14:20:15.737 INFO:tasks.rados.rados.0.smithi026.stdout:353: read oid 18 snap -1 2024-09-13T14:20:15.737 INFO:tasks.rados.rados.0.smithi026.stdout:353: expect (ObjNum 104 snap 17 seq_num 104) 2024-09-13T14:20:15.738 INFO:tasks.rados.rados.0.smithi026.stdout:354: snap_remove snap 22 2024-09-13T14:20:16.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:15 smithi026 ceph-mon[39182]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:20:16.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:15 smithi026 ceph-mon[27770]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:20:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:15 smithi145 ceph-mon[31057]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:20:16.739 INFO:tasks.rados.rados.0.smithi026.stdout:349: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:20:16.739 INFO:tasks.rados.rados.0.smithi026.stdout:349: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:20:16.739 INFO:tasks.rados.rados.0.smithi026.stdout:349: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:20:16.739 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 76 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:20:16.739 INFO:tasks.rados.rados.0.smithi026.stdout:351: expect (ObjNum 79 snap 10 seq_num 79) 2024-09-13T14:20:16.820 INFO:tasks.rados.rados.0.smithi026.stdout:348: expect (ObjNum 123 snap 28 seq_num 123) 2024-09-13T14:20:16.900 INFO:tasks.rados.rados.0.smithi026.stdout:353: expect (ObjNum 104 snap 17 seq_num 104) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:348: done (5 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:349: done (4 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:350: done (3 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:351: done (2 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:353: done (1 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:354: done (0 left) 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 30 2024-09-13T14:20:17.020 INFO:tasks.rados.rados.0.smithi026.stdout:355: seq_num 130 ranges {1687552=581632,2269184=614400,2883584=598016,3481600=393216} 2024-09-13T14:20:17.032 INFO:tasks.rados.rados.0.smithi026.stdout:355: writing smithi02683976-46 from 1687552 to 2269184 tid 1 2024-09-13T14:20:17.035 INFO:tasks.rados.rados.0.smithi026.stdout:355: writing smithi02683976-46 from 2269184 to 2883584 tid 2 2024-09-13T14:20:17.039 INFO:tasks.rados.rados.0.smithi026.stdout:355: writing smithi02683976-46 from 2883584 to 3481600 tid 3 2024-09-13T14:20:17.042 INFO:tasks.rados.rados.0.smithi026.stdout:355: writing smithi02683976-46 from 3481600 to 3874816 tid 4 2024-09-13T14:20:17.042 INFO:tasks.rados.rados.0.smithi026.stdout:356: delete oid 15 current snap is 30 2024-09-13T14:20:17.042 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 1 to smithi02683976-46 2024-09-13T14:20:17.042 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 2 to smithi02683976-46 2024-09-13T14:20:17.042 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 3 to smithi02683976-46 2024-09-13T14:20:17.044 INFO:tasks.rados.rados.0.smithi026.stdout:356: done (1 left) 2024-09-13T14:20:17.045 INFO:tasks.rados.rados.0.smithi026.stdout:357: rollback oid 45 current snap is 30 2024-09-13T14:20:17.045 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 24 2024-09-13T14:20:17.045 INFO:tasks.rados.rados.0.smithi026.stdout:358: rmattr oid 9 current snap is 30 2024-09-13T14:20:17.045 INFO:tasks.rados.rados.0.smithi026.stdout:359: snap_remove snap 20 2024-09-13T14:20:17.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:16 smithi026 ceph-mon[27770]: pgmap v188: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:17.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:16 smithi026 ceph-mon[27770]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:20:17.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:16 smithi026 ceph-mon[39182]: pgmap v188: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:16 smithi026 ceph-mon[39182]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:20:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:16 smithi145 ceph-mon[31057]: pgmap v188: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1016 KiB/s wr, 12 op/s 2024-09-13T14:20:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:16 smithi145 ceph-mon[31057]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:359: done (3 left) 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:360: read oid 30 snap -1 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:360: expect (ObjNum 91 snap 15 seq_num 91) 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:361: delete oid 37 current snap is 30 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 4 to smithi02683976-46 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 0 (ObjNum 129 snap 0 seq_num 519413576) dirty dne 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 5 to smithi02683976-46 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:355: finishing write tid 6 to smithi02683976-46 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 93 (ObjNum 130 snap 30 seq_num 130) dirty exists 2024-09-13T14:20:17.749 INFO:tasks.rados.rados.0.smithi026.stdout:355: left oid 46 (ObjNum 130 snap 30 seq_num 130) 2024-09-13T14:20:17.750 INFO:tasks.rados.rados.0.smithi026.stdout:357: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:20:17.750 INFO:tasks.rados.rados.0.smithi026.stdout:357: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:20:17.750 INFO:tasks.rados.rados.0.smithi026.stdout:357: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:20:17.750 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 46 (ObjNum 107 snap 22 seq_num 107) dirty exists 2024-09-13T14:20:17.756 INFO:tasks.rados.rados.0.smithi026.stdout:355: done (4 left) 2024-09-13T14:20:17.756 INFO:tasks.rados.rados.0.smithi026.stdout:357: done (3 left) 2024-09-13T14:20:17.756 INFO:tasks.rados.rados.0.smithi026.stdout:358: done (2 left) 2024-09-13T14:20:17.757 INFO:tasks.rados.rados.0.smithi026.stdout:361: done (1 left) 2024-09-13T14:20:17.757 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 30 2024-09-13T14:20:17.757 INFO:tasks.rados.rados.0.smithi026.stdout:362: seq_num 133 ranges {1318912=630784,1949696=663552,2613248=770048,3383296=172032} 2024-09-13T14:20:17.769 INFO:tasks.rados.rados.0.smithi026.stdout:362: writing smithi02683976-10 from 1318912 to 1949696 tid 1 2024-09-13T14:20:17.773 INFO:tasks.rados.rados.0.smithi026.stdout:362: writing smithi02683976-10 from 1949696 to 2613248 tid 2 2024-09-13T14:20:17.778 INFO:tasks.rados.rados.0.smithi026.stdout:362: writing smithi02683976-10 from 2613248 to 3383296 tid 3 2024-09-13T14:20:17.779 INFO:tasks.rados.rados.0.smithi026.stdout:362: writing smithi02683976-10 from 3383296 to 3555328 tid 4 2024-09-13T14:20:17.779 INFO:tasks.rados.rados.0.smithi026.stdout:363: snap_remove snap 29 2024-09-13T14:20:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:17 smithi026 ceph-mon[27770]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:20:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:17 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:17 smithi026 ceph-mon[39182]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:20:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:17 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:17 smithi145 ceph-mon[31057]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:20:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:17 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:18.734 INFO:tasks.rados.rados.0.smithi026.stdout:363: done (2 left) 2024-09-13T14:20:18.734 INFO:tasks.rados.rados.0.smithi026.stdout:364: snap_remove snap 12 2024-09-13T14:20:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[27770]: pgmap v191: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[27770]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:20:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[27770]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:20:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[39182]: pgmap v191: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[39182]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:20:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:18 smithi026 ceph-mon[39182]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:20:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:18 smithi145 ceph-mon[31057]: pgmap v191: 121 pgs: 121 active+clean; 95 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:18 smithi145 ceph-mon[31057]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:20:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:18 smithi145 ceph-mon[31057]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:20:19.735 INFO:tasks.rados.rados.0.smithi026.stdout:364: done (2 left) 2024-09-13T14:20:19.736 INFO:tasks.rados.rados.0.smithi026.stdout:365: snap_remove snap 23 2024-09-13T14:20:20.333 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:20:20.735 INFO:tasks.rados.rados.0.smithi026.stdout:365: done (2 left) 2024-09-13T14:20:20.735 INFO:tasks.rados.rados.0.smithi026.stdout:366: snap_remove snap 25 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 2m ago 3m 15.7M - 0.25.0 c8568f914cd2 52eac4b9b3e5 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (3m) 2m ago 3m 82.6M - 9.4.7 954c08fa6188 b034aa056559 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (5m) 2m ago 5m 12.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (5m) 2m ago 5m 17.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7926da3ac6f2 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283,8765 running (10m) 2m ago 10m 446M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 90a812e595d2 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 2m ago 10m 47.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (9m) 2m ago 9m 40.8M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:20:20.871 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (8m) 2m ago 8m 39.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 2m ago 3m 13.4M - 1.5.0 0da6a335fe13 0720764634c2 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (3m) 2m ago 3m 3355k - 1.5.0 0da6a335fe13 02ae27c8e5f5 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 2m ago 8m 44.3M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (7m) 2m ago 7m 43.6M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 2m ago 7m 44.2M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 2m ago 7m 41.7M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (6m) 2m ago 6m 43.9M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (6m) 2m ago 6m 42.2M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (5m) 2m ago 5m 41.5M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (5m) 2m ago 5m 44.0M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:20:20.872 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (2m) 2m ago 4m 29.3M - 2.43.0 a07b618ecd1d 33771513b4fc 2024-09-13T14:20:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:20 smithi026 ceph-mon[27770]: pgmap v194: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:20 smithi026 ceph-mon[27770]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:20:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:20 smithi026 ceph-mon[39182]: pgmap v194: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:20 smithi026 ceph-mon[39182]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:20:21.194 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 15 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:20:21.195 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:20:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:20 smithi145 ceph-mon[31057]: pgmap v194: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:20 smithi145 ceph-mon[31057]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:20:21.465 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:20:21.465 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:20:21.465 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:20:21.465 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:20:21.465 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-09-13T14:20:21.466 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/20 daemons upgraded", 2024-09-13T14:20:21.466 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f image on host smithi145", 2024-09-13T14:20:21.466 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:20:21.466 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:20:21.746 INFO:tasks.rados.rados.0.smithi026.stdout:366: done (2 left) 2024-09-13T14:20:21.746 INFO:tasks.rados.rados.0.smithi026.stdout:367: setattr oid 43 current snap is 30 2024-09-13T14:20:21.746 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 1 to smithi02683976-10 2024-09-13T14:20:21.746 INFO:tasks.rados.rados.0.smithi026.stdout:360: expect (ObjNum 91 snap 15 seq_num 91) 2024-09-13T14:20:21.812 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 2 to smithi02683976-10 2024-09-13T14:20:21.812 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 3 to smithi02683976-10 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 4 to smithi02683976-10 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 5 to smithi02683976-10 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:362: finishing write tid 6 to smithi02683976-10 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 100 (ObjNum 133 snap 30 seq_num 133) dirty exists 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:362: left oid 10 (ObjNum 133 snap 30 seq_num 133) 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:360: done (2 left) 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:362: done (1 left) 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:368: read oid 26 snap -1 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:368: expect deleted 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 30 2024-09-13T14:20:21.813 INFO:tasks.rados.rados.0.smithi026.stdout:369: seq_num 134 ranges {2580480=647168,3227648=548864,3776512=573440,4349952=483328} 2024-09-13T14:20:21.830 INFO:tasks.rados.rados.0.smithi026.stdout:369: writing smithi02683976-31 from 2580480 to 3227648 tid 1 2024-09-13T14:20:21.833 INFO:tasks.rados.rados.0.smithi026.stdout:369: writing smithi02683976-31 from 3227648 to 3776512 tid 2 2024-09-13T14:20:21.836 INFO:tasks.rados.rados.0.smithi026.stdout:369: writing smithi02683976-31 from 3776512 to 4349952 tid 3 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:369: writing smithi02683976-31 from 4349952 to 4833280 tid 4 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:370: setattr oid 8 current snap is 30 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 187 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 1 to smithi02683976-31 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 2 to smithi02683976-31 2024-09-13T14:20:21.839 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 3 to smithi02683976-31 2024-09-13T14:20:21.840 INFO:tasks.rados.rados.0.smithi026.stdout:367: done (3 left) 2024-09-13T14:20:21.840 INFO:tasks.rados.rados.0.smithi026.stdout:368: done (2 left) 2024-09-13T14:20:21.840 INFO:tasks.rados.rados.0.smithi026.stdout:371: read oid 33 snap -1 2024-09-13T14:20:21.840 INFO:tasks.rados.rados.0.smithi026.stdout:371: expect (ObjNum 105 snap 18 seq_num 105) 2024-09-13T14:20:21.840 INFO:tasks.rados.rados.0.smithi026.stdout:372: snap_remove snap 24 2024-09-13T14:20:22.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[27770]: from='client.24881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[27770]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[27770]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1453038291' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[39182]: from='client.24881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[39182]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[39182]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:20:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:21 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1453038291' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:21 smithi145 ceph-mon[31057]: from='client.24881 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:21 smithi145 ceph-mon[31057]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:21 smithi145 ceph-mon[31057]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:20:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:21 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1453038291' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:22.752 INFO:tasks.rados.rados.0.smithi026.stdout:372: done (3 left) 2024-09-13T14:20:22.752 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 30 2024-09-13T14:20:22.752 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 4 to smithi02683976-31 2024-09-13T14:20:22.752 INFO:tasks.rados.rados.0.smithi026.stdout:373: seq_num 135 ranges {0=663552,663552=434176,1097728=786432,1884160=385024} 2024-09-13T14:20:22.756 INFO:tasks.rados.rados.0.smithi026.stdout:373: writing smithi02683976-36 from 0 to 663552 tid 1 2024-09-13T14:20:22.759 INFO:tasks.rados.rados.0.smithi026.stdout:373: writing smithi02683976-36 from 663552 to 1097728 tid 2 2024-09-13T14:20:22.765 INFO:tasks.rados.rados.0.smithi026.stdout:373: writing smithi02683976-36 from 1097728 to 1884160 tid 3 2024-09-13T14:20:22.768 INFO:tasks.rados.rados.0.smithi026.stdout:373: writing smithi02683976-36 from 1884160 to 2269184 tid 4 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:374: rmattr oid 40 current snap is 30 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:374: done (4 left) 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:375: copy_from oid 46 from oid 13 current snap is 30 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 5 to smithi02683976-31 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:369: finishing write tid 6 to smithi02683976-31 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 167 (ObjNum 134 snap 30 seq_num 134) dirty exists 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:369: left oid 31 (ObjNum 134 snap 30 seq_num 134) 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 32 (ObjNum 1217 snap 0 seq_num 519328720) dirty exists 2024-09-13T14:20:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:371: expect (ObjNum 105 snap 18 seq_num 105) 2024-09-13T14:20:22.842 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 1 to smithi02683976-36 2024-09-13T14:20:22.842 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 2 to smithi02683976-36 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 3 to smithi02683976-36 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 4 to smithi02683976-36 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 5 to smithi02683976-36 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: finishing write tid 6 to smithi02683976-36 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 205 (ObjNum 135 snap 30 seq_num 135) dirty exists 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: left oid 36 (ObjNum 135 snap 30 seq_num 135) 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:375: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:375: finishing copy_from to smithi02683976-46 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 147 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:369: done (4 left) 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:370: done (3 left) 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:371: done (2 left) 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:373: done (1 left) 2024-09-13T14:20:22.843 INFO:tasks.rados.rados.0.smithi026.stdout:375: done (0 left) 2024-09-13T14:20:22.844 INFO:tasks.rados.rados.0.smithi026.stdout:376: delete oid 49 current snap is 30 2024-09-13T14:20:22.844 INFO:tasks.rados.rados.0.smithi026.stdout:376: done (0 left) 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:377: copy_from oid 35 from oid 7 current snap is 30 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:378: read oid 11 snap -1 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:378: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:379: copy_from oid 1 from oid 17 current snap is 30 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:380: copy_from oid 24 from oid 22 current snap is 30 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 30 2024-09-13T14:20:22.845 INFO:tasks.rados.rados.0.smithi026.stdout:381: seq_num 137 ranges {1638400=704512,2342912=655360,2998272=753664,3751936=196608} 2024-09-13T14:20:22.859 INFO:tasks.rados.rados.0.smithi026.stdout:381: writing smithi02683976-6 from 1638400 to 2342912 tid 1 2024-09-13T14:20:22.863 INFO:tasks.rados.rados.0.smithi026.stdout:381: writing smithi02683976-6 from 2342912 to 2998272 tid 2 2024-09-13T14:20:22.867 INFO:tasks.rados.rados.0.smithi026.stdout:381: writing smithi02683976-6 from 2998272 to 3751936 tid 3 2024-09-13T14:20:22.869 INFO:tasks.rados.rados.0.smithi026.stdout:381: writing smithi02683976-6 from 3751936 to 3948544 tid 4 2024-09-13T14:20:22.869 INFO:tasks.rados.rados.0.smithi026.stdout:382: read oid 37 snap -1 2024-09-13T14:20:22.869 INFO:tasks.rados.rados.0.smithi026.stdout:382: expect deleted 2024-09-13T14:20:22.869 INFO:tasks.rados.rados.0.smithi026.stdout:383: read oid 5 snap -1 2024-09-13T14:20:22.869 INFO:tasks.rados.rados.0.smithi026.stdout:383: expect (ObjNum 117 snap 25 seq_num 117) 2024-09-13T14:20:22.870 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 30 2024-09-13T14:20:22.870 INFO:tasks.rados.rados.0.smithi026.stdout:384: seq_num 138 ranges {2547712=720896,3268608=540672,3809280=557056,4366336=507904} 2024-09-13T14:20:22.887 INFO:tasks.rados.rados.0.smithi026.stdout:384: writing smithi02683976-32 from 2547712 to 3268608 tid 1 2024-09-13T14:20:22.890 INFO:tasks.rados.rados.0.smithi026.stdout:384: writing smithi02683976-32 from 3268608 to 3809280 tid 2 2024-09-13T14:20:22.894 INFO:tasks.rados.rados.0.smithi026.stdout:384: writing smithi02683976-32 from 3809280 to 4366336 tid 3 2024-09-13T14:20:22.897 INFO:tasks.rados.rados.0.smithi026.stdout:384: writing smithi02683976-32 from 4366336 to 4874240 tid 4 2024-09-13T14:20:22.897 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 30 2024-09-13T14:20:22.897 INFO:tasks.rados.rados.0.smithi026.stdout:385: seq_num 139 ranges {2023424=737280,2760704=425984,3186688=761856,3948544=417792} 2024-09-13T14:20:22.912 INFO:tasks.rados.rados.0.smithi026.stdout:385: writing smithi02683976-30 from 2023424 to 2760704 tid 1 2024-09-13T14:20:22.914 INFO:tasks.rados.rados.0.smithi026.stdout:385: writing smithi02683976-30 from 2760704 to 3186688 tid 2 2024-09-13T14:20:22.919 INFO:tasks.rados.rados.0.smithi026.stdout:385: writing smithi02683976-30 from 3186688 to 3948544 tid 3 2024-09-13T14:20:22.922 INFO:tasks.rados.rados.0.smithi026.stdout:385: writing smithi02683976-30 from 3948544 to 4366336 tid 4 2024-09-13T14:20:22.922 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 30 2024-09-13T14:20:22.922 INFO:tasks.rados.rados.0.smithi026.stdout:377: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:20:22.922 INFO:tasks.rados.rados.0.smithi026.stdout:378: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:20:22.926 INFO:tasks.rados.rados.0.smithi026.stdout:377: finishing copy_from to smithi02683976-35 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 6 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:380: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:380: finishing copy_from to smithi02683976-24 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 76 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:379: finishing copy_from racing read to smithi02683976-1 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:379: finishing copy_from to smithi02683976-1 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 58 (ObjNum 88 snap 14 seq_num 88) dirty exists 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 1 to smithi02683976-6 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 2 to smithi02683976-6 2024-09-13T14:20:22.927 INFO:tasks.rados.rados.0.smithi026.stdout:386: seq_num 140 ranges {425984=753664,1179648=704512,1884160=524288,2408448=376832} 2024-09-13T14:20:22.933 INFO:tasks.rados.rados.0.smithi026.stdout:386: writing smithi02683976-27 from 425984 to 1179648 tid 1 2024-09-13T14:20:22.937 INFO:tasks.rados.rados.0.smithi026.stdout:386: writing smithi02683976-27 from 1179648 to 1884160 tid 2 2024-09-13T14:20:22.940 INFO:tasks.rados.rados.0.smithi026.stdout:386: writing smithi02683976-27 from 1884160 to 2408448 tid 3 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:386: writing smithi02683976-27 from 2408448 to 2785280 tid 4 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:377: done (9 left) 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:378: done (8 left) 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:379: done (7 left) 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:380: done (6 left) 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:387: delete oid 3 current snap is 30 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 3 to smithi02683976-6 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 4 to smithi02683976-6 2024-09-13T14:20:22.943 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 5 to smithi02683976-6 2024-09-13T14:20:22.944 INFO:tasks.rados.rados.0.smithi026.stdout:381: finishing write tid 6 to smithi02683976-6 2024-09-13T14:20:22.944 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 174 (ObjNum 137 snap 30 seq_num 137) dirty exists 2024-09-13T14:20:22.944 INFO:tasks.rados.rados.0.smithi026.stdout:381: left oid 6 (ObjNum 137 snap 30 seq_num 137) 2024-09-13T14:20:22.944 INFO:tasks.rados.rados.0.smithi026.stdout:383: expect (ObjNum 117 snap 25 seq_num 117) 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 1 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 2 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 3 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 4 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 5 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: finishing write tid 6 to smithi02683976-32 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 180 (ObjNum 138 snap 30 seq_num 138) dirty exists 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:384: left oid 32 (ObjNum 138 snap 30 seq_num 138) 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 1 to smithi02683976-30 2024-09-13T14:20:23.090 INFO:tasks.rados.rados.0.smithi026.stdout:381: done (6 left) 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:382: done (5 left) 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:383: done (4 left) 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:384: done (3 left) 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:387: done (2 left) 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 30 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 2 to smithi02683976-30 2024-09-13T14:20:23.091 INFO:tasks.rados.rados.0.smithi026.stdout:388: seq_num 142 ranges {106496=786432,892928=475136,1368064=540672,1908736=589824} 2024-09-13T14:20:23.094 INFO:tasks.rados.rados.0.smithi026.stdout:388: writing smithi02683976-7 from 106496 to 892928 tid 1 2024-09-13T14:20:23.097 INFO:tasks.rados.rados.0.smithi026.stdout:388: writing smithi02683976-7 from 892928 to 1368064 tid 2 2024-09-13T14:20:23.100 INFO:tasks.rados.rados.0.smithi026.stdout:388: writing smithi02683976-7 from 1368064 to 1908736 tid 3 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:388: writing smithi02683976-7 from 1908736 to 2498560 tid 4 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:389: setattr oid 13 current snap is 30 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 3 to smithi02683976-30 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 4 to smithi02683976-30 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 5 to smithi02683976-30 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:385: finishing write tid 6 to smithi02683976-30 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 108 (ObjNum 139 snap 30 seq_num 139) dirty exists 2024-09-13T14:20:23.105 INFO:tasks.rados.rados.0.smithi026.stdout:385: left oid 30 (ObjNum 139 snap 30 seq_num 139) 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 1 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 2 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 3 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 4 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 5 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: finishing write tid 6 to smithi02683976-27 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 218 (ObjNum 140 snap 30 seq_num 140) dirty exists 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:386: left oid 27 (ObjNum 140 snap 30 seq_num 140) 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 1 to smithi02683976-7 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 2 to smithi02683976-7 2024-09-13T14:20:23.106 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 3 to smithi02683976-7 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:385: done (3 left) 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:386: done (2 left) 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:390: read oid 15 snap -1 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:390: expect deleted 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:391: snap_create 2024-09-13T14:20:23.107 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 4 to smithi02683976-7 2024-09-13T14:20:23.108 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 5 to smithi02683976-7 2024-09-13T14:20:23.108 INFO:tasks.rados.rados.0.smithi026.stdout:388: finishing write tid 6 to smithi02683976-7 2024-09-13T14:20:23.108 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 15 (ObjNum 142 snap 30 seq_num 142) dirty exists 2024-09-13T14:20:23.108 INFO:tasks.rados.rados.0.smithi026.stdout:388: left oid 7 (ObjNum 142 snap 30 seq_num 142) 2024-09-13T14:20:23.108 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 194 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[27770]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[27770]: pgmap v197: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[27770]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[27770]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:20:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[39182]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[39182]: pgmap v197: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[39182]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:22 smithi026 ceph-mon[39182]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:20:23.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:20:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:22 smithi145 ceph-mon[31057]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:22 smithi145 ceph-mon[31057]: pgmap v197: 121 pgs: 121 active+clean; 103 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-13T14:20:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:22 smithi145 ceph-mon[31057]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:22 smithi145 ceph-mon[31057]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:20:23.756 INFO:tasks.rados.rados.0.smithi026.stdout:388: done (3 left) 2024-09-13T14:20:23.756 INFO:tasks.rados.rados.0.smithi026.stdout:389: done (2 left) 2024-09-13T14:20:23.756 INFO:tasks.rados.rados.0.smithi026.stdout:390: done (1 left) 2024-09-13T14:20:23.756 INFO:tasks.rados.rados.0.smithi026.stdout:391: done (0 left) 2024-09-13T14:20:23.756 INFO:tasks.rados.rados.0.smithi026.stdout:392: delete oid 27 current snap is 31 2024-09-13T14:20:23.759 INFO:tasks.rados.rados.0.smithi026.stdout:392: done (0 left) 2024-09-13T14:20:23.759 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 31 2024-09-13T14:20:23.760 INFO:tasks.rados.rados.0.smithi026.stdout:393: seq_num 144 ranges {0=16384} 2024-09-13T14:20:23.760 INFO:tasks.rados.rados.0.smithi026.stdout:393: writing smithi02683976-3 from 0 to 16384 tid 1 2024-09-13T14:20:23.760 INFO:tasks.rados.rados.0.smithi026.stdout:394: snap_create 2024-09-13T14:20:23.761 INFO:tasks.rados.rados.0.smithi026.stdout:393: finishing write tid 1 to smithi02683976-3 2024-09-13T14:20:23.762 INFO:tasks.rados.rados.0.smithi026.stdout:393: finishing write tid 2 to smithi02683976-3 2024-09-13T14:20:23.762 INFO:tasks.rados.rados.0.smithi026.stdout:393: finishing write tid 3 to smithi02683976-3 2024-09-13T14:20:23.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 223 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:23.763 INFO:tasks.rados.rados.0.smithi026.stdout:393: left oid 3 (ObjNum 144 snap 31 seq_num 144) 2024-09-13T14:20:24.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:23 smithi026 ceph-mon[27770]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:20:24.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:23 smithi026 ceph-mon[39182]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:20:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:23 smithi145 ceph-mon[31057]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:20:24.761 INFO:tasks.rados.rados.0.smithi026.stdout:393: done (1 left) 2024-09-13T14:20:24.762 INFO:tasks.rados.rados.0.smithi026.stdout:394: done (0 left) 2024-09-13T14:20:24.762 INFO:tasks.rados.rados.0.smithi026.stdout:395: rollback oid 27 current snap is 32 2024-09-13T14:20:24.762 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 31 2024-09-13T14:20:24.762 INFO:tasks.rados.rados.0.smithi026.stdout:396: setattr oid 20 current snap is 32 2024-09-13T14:20:24.762 INFO:tasks.rados.rados.0.smithi026.stdout:397: snap_create 2024-09-13T14:20:24.766 INFO:tasks.rados.rados.0.smithi026.stdout:395: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:20:24.766 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 221 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:20:24.767 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 195 (ObjNum 177 snap 0 seq_num 0) dirty exists 2024-09-13T14:20:25.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:24 smithi026 ceph-mon[27770]: pgmap v200: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:24 smithi026 ceph-mon[27770]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:20:25.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:24 smithi026 ceph-mon[39182]: pgmap v200: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:25.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:24 smithi026 ceph-mon[39182]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:20:25.065 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:24 smithi145 ceph-mon[31057]: pgmap v200: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:25.065 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:24 smithi145 ceph-mon[31057]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:20:25.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:25 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:25] "GET /metrics HTTP/1.1" 200 240003 "" "Prometheus/2.43.0" 2024-09-13T14:20:25.767 INFO:tasks.rados.rados.0.smithi026.stdout:395: done (2 left) 2024-09-13T14:20:25.767 INFO:tasks.rados.rados.0.smithi026.stdout:396: done (1 left) 2024-09-13T14:20:25.767 INFO:tasks.rados.rados.0.smithi026.stdout:397: done (0 left) 2024-09-13T14:20:25.767 INFO:tasks.rados.rados.0.smithi026.stdout:398: read oid 46 snap -1 2024-09-13T14:20:25.768 INFO:tasks.rados.rados.0.smithi026.stdout:398: expect (ObjNum 127 snap 29 seq_num 127) 2024-09-13T14:20:25.768 INFO:tasks.rados.rados.0.smithi026.stdout:399: snap_create 2024-09-13T14:20:25.796 INFO:tasks.rados.rados.0.smithi026.stdout:398: expect (ObjNum 127 snap 29 seq_num 127) 2024-09-13T14:20:26.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:25 smithi026 ceph-mon[27770]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:20:26.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:25 smithi026 ceph-mon[39182]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:20:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:25 smithi145 ceph-mon[31057]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:20:26.768 INFO:tasks.rados.rados.0.smithi026.stdout:398: done (1 left) 2024-09-13T14:20:26.769 INFO:tasks.rados.rados.0.smithi026.stdout:399: done (0 left) 2024-09-13T14:20:26.769 INFO:tasks.rados.rados.0.smithi026.stdout:400: delete oid 32 current snap is 34 2024-09-13T14:20:26.776 INFO:tasks.rados.rados.0.smithi026.stdout:400: done (0 left) 2024-09-13T14:20:26.776 INFO:tasks.rados.rados.0.smithi026.stdout:401: read oid 29 snap 32 2024-09-13T14:20:26.776 INFO:tasks.rados.rados.0.smithi026.stdout:401: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:20:26.776 INFO:tasks.rados.rados.0.smithi026.stdout:402: delete oid 1 current snap is 34 2024-09-13T14:20:26.779 INFO:tasks.rados.rados.0.smithi026.stdout:402: done (1 left) 2024-09-13T14:20:26.779 INFO:tasks.rados.rados.0.smithi026.stdout:403: copy_from oid 37 from oid 3 current snap is 34 2024-09-13T14:20:26.779 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 34 2024-09-13T14:20:26.779 INFO:tasks.rados.rados.0.smithi026.stdout:404: seq_num 147 ranges {4366336=65536} 2024-09-13T14:20:26.803 INFO:tasks.rados.rados.0.smithi026.stdout:404: writing smithi02683976-30 from 4366336 to 4431872 tid 1 2024-09-13T14:20:26.803 INFO:tasks.rados.rados.0.smithi026.stdout:405: rmattr oid 16 current snap is 34 2024-09-13T14:20:26.803 INFO:tasks.rados.rados.0.smithi026.stdout:405: done (3 left) 2024-09-13T14:20:26.803 INFO:tasks.rados.rados.0.smithi026.stdout:406: delete oid 35 current snap is 34 2024-09-13T14:20:26.804 INFO:tasks.rados.rados.0.smithi026.stdout:401: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:403: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:403: finishing copy_from to smithi02683976-37 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 223 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:404: finishing write tid 1 to smithi02683976-30 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:404: finishing write tid 2 to smithi02683976-30 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:404: finishing write tid 3 to smithi02683976-30 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 111 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:404: left oid 30 (ObjNum 147 snap 34 seq_num 147) 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:401: done (3 left) 2024-09-13T14:20:26.825 INFO:tasks.rados.rados.0.smithi026.stdout:403: done (2 left) 2024-09-13T14:20:26.826 INFO:tasks.rados.rados.0.smithi026.stdout:404: done (1 left) 2024-09-13T14:20:26.826 INFO:tasks.rados.rados.0.smithi026.stdout:406: done (0 left) 2024-09-13T14:20:26.826 INFO:tasks.rados.rados.0.smithi026.stdout:407: read oid 24 snap -1 2024-09-13T14:20:26.826 INFO:tasks.rados.rados.0.smithi026.stdout:407: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:20:26.826 INFO:tasks.rados.rados.0.smithi026.stdout:408: snap_create 2024-09-13T14:20:26.831 INFO:tasks.rados.rados.0.smithi026.stdout:407: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:20:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:26 smithi026 ceph-mon[27770]: pgmap v203: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:26 smithi026 ceph-mon[27770]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:20:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:26 smithi026 ceph-mon[39182]: pgmap v203: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:26 smithi026 ceph-mon[39182]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:20:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:26 smithi145 ceph-mon[31057]: pgmap v203: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 30 op/s 2024-09-13T14:20:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:26 smithi145 ceph-mon[31057]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:20:27.778 INFO:tasks.rados.rados.0.smithi026.stdout:407: done (1 left) 2024-09-13T14:20:27.778 INFO:tasks.rados.rados.0.smithi026.stdout:408: done (0 left) 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:409: rmattr oid 6 current snap is 35 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:410: rollback oid 3 current snap is 35 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 33 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:411: rollback oid 16 current snap is 35 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 30 2024-09-13T14:20:27.779 INFO:tasks.rados.rados.0.smithi026.stdout:412: snap_create 2024-09-13T14:20:27.783 INFO:tasks.rados.rados.0.smithi026.stdout:411: finishing rollback tid 0 to smithi02683976-16 2024-09-13T14:20:27.783 INFO:tasks.rados.rados.0.smithi026.stdout:411: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:20:27.783 INFO:tasks.rados.rados.0.smithi026.stdout:411: finishing rollback tid 2 to smithi02683976-16 2024-09-13T14:20:27.783 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 199 (ObjNum 109 snap 24 seq_num 109) dirty exists 2024-09-13T14:20:27.784 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 176 (ObjNum 137 snap 30 seq_num 137) dirty exists 2024-09-13T14:20:27.785 INFO:tasks.rados.rados.0.smithi026.stdout:410: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:20:27.785 INFO:tasks.rados.rados.0.smithi026.stdout:410: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:20:27.785 INFO:tasks.rados.rados.0.smithi026.stdout:410: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:20:27.785 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 232 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:27 smithi026 ceph-mon[39182]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:20:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:27 smithi026 ceph-mon[27770]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:20:28.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:27 smithi145 ceph-mon[31057]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:409: done (3 left) 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:410: done (2 left) 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:411: done (1 left) 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:412: done (0 left) 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:413: rollback oid 39 current snap is 36 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 32 2024-09-13T14:20:28.735 INFO:tasks.rados.rados.0.smithi026.stdout:414: rollback oid 21 current snap is 36 2024-09-13T14:20:28.736 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 31 2024-09-13T14:20:28.736 INFO:tasks.rados.rados.0.smithi026.stdout:415: delete oid 24 current snap is 36 2024-09-13T14:20:28.740 INFO:tasks.rados.rados.0.smithi026.stdout:413: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:414: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:413: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:414: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:414: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 180 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:413: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:20:28.742 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 121 (ObjNum 57 snap 4 seq_num 57) dirty exists 2024-09-13T14:20:28.743 INFO:tasks.rados.rados.0.smithi026.stdout:413: done (2 left) 2024-09-13T14:20:28.743 INFO:tasks.rados.rados.0.smithi026.stdout:414: done (1 left) 2024-09-13T14:20:28.743 INFO:tasks.rados.rados.0.smithi026.stdout:415: done (0 left) 2024-09-13T14:20:28.743 INFO:tasks.rados.rados.0.smithi026.stdout:416: delete oid 47 current snap is 36 2024-09-13T14:20:28.747 INFO:tasks.rados.rados.0.smithi026.stdout:416: done (0 left) 2024-09-13T14:20:28.747 INFO:tasks.rados.rados.0.smithi026.stdout:417: read oid 33 snap 32 2024-09-13T14:20:28.747 INFO:tasks.rados.rados.0.smithi026.stdout:417: expect (ObjNum 105 snap 18 seq_num 105) 2024-09-13T14:20:28.747 INFO:tasks.rados.rados.0.smithi026.stdout:418: setattr oid 15 current snap is 36 2024-09-13T14:20:28.749 INFO:tasks.rados.rados.0.smithi026.stdout:419: setattr oid 29 current snap is 36 2024-09-13T14:20:28.750 INFO:tasks.rados.rados.0.smithi026.stdout:420: delete oid 34 current snap is 36 2024-09-13T14:20:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 112 (ObjNum 33 snap 0 seq_num 519460160) dirty exists 2024-09-13T14:20:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:418: done (3 left) 2024-09-13T14:20:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:420: done (2 left) 2024-09-13T14:20:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 36 2024-09-13T14:20:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:421: seq_num 152 ranges {2269184=147456} 2024-09-13T14:20:28.766 INFO:tasks.rados.rados.0.smithi026.stdout:421: writing smithi02683976-36 from 2269184 to 2416640 tid 1 2024-09-13T14:20:28.766 INFO:tasks.rados.rados.0.smithi026.stdout:422: snap_remove snap 35 2024-09-13T14:20:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[27770]: pgmap v206: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[27770]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:20:29.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[27770]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:20:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[39182]: pgmap v206: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[39182]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:20:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:28 smithi026 ceph-mon[39182]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:20:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:28 smithi145 ceph-mon[31057]: pgmap v206: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:28 smithi145 ceph-mon[31057]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:20:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:28 smithi145 ceph-mon[31057]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:20:29.737 INFO:tasks.rados.rados.0.smithi026.stdout:422: done (3 left) 2024-09-13T14:20:29.737 INFO:tasks.rados.rados.0.smithi026.stdout:423: delete oid 12 current snap is 36 2024-09-13T14:20:29.737 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 201 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:20:29.737 INFO:tasks.rados.rados.0.smithi026.stdout:417: expect (ObjNum 105 snap 18 seq_num 105) 2024-09-13T14:20:29.809 INFO:tasks.rados.rados.0.smithi026.stdout:421: finishing write tid 1 to smithi02683976-36 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:421: finishing write tid 2 to smithi02683976-36 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:421: finishing write tid 3 to smithi02683976-36 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 235 (ObjNum 152 snap 36 seq_num 152) dirty exists 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:421: left oid 36 (ObjNum 152 snap 36 seq_num 152) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:417: done (3 left) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:419: done (2 left) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:421: done (1 left) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:423: done (0 left) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:424: rollback oid 9 current snap is 36 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 31 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:425: read oid 11 snap 30 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:425: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:20:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 36 2024-09-13T14:20:29.811 INFO:tasks.rados.rados.0.smithi026.stdout:426: seq_num 154 ranges {2547712=180224} 2024-09-13T14:20:29.824 INFO:tasks.rados.rados.0.smithi026.stdout:426: writing smithi02683976-23 from 2547712 to 2727936 tid 1 2024-09-13T14:20:29.824 INFO:tasks.rados.rados.0.smithi026.stdout:427: delete oid 7 current snap is 36 2024-09-13T14:20:29.824 INFO:tasks.rados.rados.0.smithi026.stdout:424: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:20:29.825 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-13T14:20:29.825 INFO:tasks.rados.rados.0.smithi026.stdout:425: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:424: done (3 left) 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:425: done (2 left) 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:427: done (1 left) 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:428: read oid 26 snap 31 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:428: expect deleted 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:429: read oid 47 snap -1 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:429: expect deleted 2024-09-13T14:20:29.832 INFO:tasks.rados.rados.0.smithi026.stdout:430: read oid 41 snap -1 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:430: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:431: copy_from oid 28 from oid 44 current snap is 36 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:426: finishing write tid 1 to smithi02683976-23 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:426: finishing write tid 2 to smithi02683976-23 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:426: finishing write tid 3 to smithi02683976-23 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 185 (ObjNum 154 snap 36 seq_num 154) dirty exists 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:426: left oid 23 (ObjNum 154 snap 36 seq_num 154) 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:426: done (4 left) 2024-09-13T14:20:29.833 INFO:tasks.rados.rados.0.smithi026.stdout:432: snap_remove snap 32 2024-09-13T14:20:30.781 INFO:tasks.rados.rados.0.smithi026.stdout:432: done (4 left) 2024-09-13T14:20:30.781 INFO:tasks.rados.rados.0.smithi026.stdout:433: rollback oid 46 current snap is 36 2024-09-13T14:20:30.782 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 33 2024-09-13T14:20:30.782 INFO:tasks.rados.rados.0.smithi026.stdout:430: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:431: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:431: finishing copy_from to smithi02683976-28 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 85 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:433: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:433: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:20:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:428: done (4 left) 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:429: done (3 left) 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:430: done (2 left) 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:431: done (1 left) 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 36 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:433: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 150 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:30.834 INFO:tasks.rados.rados.0.smithi026.stdout:434: seq_num 156 ranges {1687552=221184} 2024-09-13T14:20:30.844 INFO:tasks.rados.rados.0.smithi026.stdout:434: writing smithi02683976-14 from 1687552 to 1908736 tid 1 2024-09-13T14:20:30.845 INFO:tasks.rados.rados.0.smithi026.stdout:433: done (1 left) 2024-09-13T14:20:30.845 INFO:tasks.rados.rados.0.smithi026.stdout:435: copy_from oid 47 from oid 8 current snap is 36 2024-09-13T14:20:30.845 INFO:tasks.rados.rados.0.smithi026.stdout:436: snap_create 2024-09-13T14:20:30.846 INFO:tasks.rados.rados.0.smithi026.stdout:435: finishing copy_from racing read to smithi02683976-47 2024-09-13T14:20:30.847 INFO:tasks.rados.rados.0.smithi026.stdout:435: finishing copy_from to smithi02683976-47 2024-09-13T14:20:30.847 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 32 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:20:30.847 INFO:tasks.rados.rados.0.smithi026.stdout:434: finishing write tid 1 to smithi02683976-14 2024-09-13T14:20:30.847 INFO:tasks.rados.rados.0.smithi026.stdout:434: finishing write tid 2 to smithi02683976-14 2024-09-13T14:20:30.848 INFO:tasks.rados.rados.0.smithi026.stdout:434: finishing write tid 3 to smithi02683976-14 2024-09-13T14:20:30.848 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 192 (ObjNum 156 snap 36 seq_num 156) dirty exists 2024-09-13T14:20:30.848 INFO:tasks.rados.rados.0.smithi026.stdout:434: left oid 14 (ObjNum 156 snap 36 seq_num 156) 2024-09-13T14:20:31.137 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:30 smithi145 ceph-mon[31057]: pgmap v209: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:31.137 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:30 smithi145 ceph-mon[31057]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:20:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:30 smithi026 ceph-mon[39182]: pgmap v209: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:30 smithi026 ceph-mon[39182]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:20:31.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:30 smithi026 ceph-mon[27770]: pgmap v209: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:30 smithi026 ceph-mon[27770]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:20:31.786 INFO:tasks.rados.rados.0.smithi026.stdout:434: done (2 left) 2024-09-13T14:20:31.786 INFO:tasks.rados.rados.0.smithi026.stdout:435: done (1 left) 2024-09-13T14:20:31.786 INFO:tasks.rados.rados.0.smithi026.stdout:436: done (0 left) 2024-09-13T14:20:31.786 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 37 2024-09-13T14:20:31.786 INFO:tasks.rados.rados.0.smithi026.stdout:437: seq_num 157 ranges {0=237568} 2024-09-13T14:20:31.787 INFO:tasks.rados.rados.0.smithi026.stdout:437: writing smithi02683976-12 from 0 to 237568 tid 1 2024-09-13T14:20:31.787 INFO:tasks.rados.rados.0.smithi026.stdout:438: read oid 9 snap -1 2024-09-13T14:20:31.787 INFO:tasks.rados.rados.0.smithi026.stdout:438: expect deleted 2024-09-13T14:20:31.788 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 37 2024-09-13T14:20:31.788 INFO:tasks.rados.rados.0.smithi026.stdout:439: seq_num 158 ranges {0=253952} 2024-09-13T14:20:31.789 INFO:tasks.rados.rados.0.smithi026.stdout:439: writing smithi02683976-26 from 0 to 253952 tid 1 2024-09-13T14:20:31.789 INFO:tasks.rados.rados.0.smithi026.stdout:440: snap_create 2024-09-13T14:20:31.796 INFO:tasks.rados.rados.0.smithi026.stdout:437: finishing write tid 1 to smithi02683976-12 2024-09-13T14:20:31.796 INFO:tasks.rados.rados.0.smithi026.stdout:437: finishing write tid 2 to smithi02683976-12 2024-09-13T14:20:31.797 INFO:tasks.rados.rados.0.smithi026.stdout:439: finishing write tid 1 to smithi02683976-26 2024-09-13T14:20:31.797 INFO:tasks.rados.rados.0.smithi026.stdout:439: finishing write tid 2 to smithi02683976-26 2024-09-13T14:20:31.798 INFO:tasks.rados.rados.0.smithi026.stdout:437: finishing write tid 3 to smithi02683976-12 2024-09-13T14:20:31.798 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 119 (ObjNum 157 snap 37 seq_num 157) dirty exists 2024-09-13T14:20:31.799 INFO:tasks.rados.rados.0.smithi026.stdout:437: left oid 12 (ObjNum 157 snap 37 seq_num 157) 2024-09-13T14:20:31.799 INFO:tasks.rados.rados.0.smithi026.stdout:439: finishing write tid 3 to smithi02683976-26 2024-09-13T14:20:31.799 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 121 (ObjNum 158 snap 37 seq_num 158) dirty exists 2024-09-13T14:20:31.799 INFO:tasks.rados.rados.0.smithi026.stdout:439: left oid 26 (ObjNum 158 snap 37 seq_num 158) 2024-09-13T14:20:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:31 smithi145 ceph-mon[31057]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:20:32.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:31 smithi026 ceph-mon[27770]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:20:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:31 smithi026 ceph-mon[39182]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:437: done (3 left) 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:438: done (2 left) 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:439: done (1 left) 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:440: done (0 left) 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:441: copy_from oid 15 from oid 40 current snap is 38 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:442: read oid 7 snap 30 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:442: expect (ObjNum 142 snap 30 seq_num 142) 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:443: rollback oid 31 current snap is 38 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 37 2024-09-13T14:20:32.788 INFO:tasks.rados.rados.0.smithi026.stdout:444: delete oid 35 current snap is 38 2024-09-13T14:20:32.796 INFO:tasks.rados.rados.0.smithi026.stdout:444: done (3 left) 2024-09-13T14:20:32.796 INFO:tasks.rados.rados.0.smithi026.stdout:445: read oid 38 snap -1 2024-09-13T14:20:32.796 INFO:tasks.rados.rados.0.smithi026.stdout:445: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:20:32.796 INFO:tasks.rados.rados.0.smithi026.stdout:446: delete oid 26 current snap is 38 2024-09-13T14:20:32.797 INFO:tasks.rados.rados.0.smithi026.stdout:443: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:20:32.798 INFO:tasks.rados.rados.0.smithi026.stdout:446: done (4 left) 2024-09-13T14:20:32.798 INFO:tasks.rados.rados.0.smithi026.stdout:447: rmattr oid 23 current snap is 38 2024-09-13T14:20:32.799 INFO:tasks.rados.rados.0.smithi026.stdout:447: done (4 left) 2024-09-13T14:20:32.799 INFO:tasks.rados.rados.0.smithi026.stdout:448: copy_from oid 33 from oid 10 current snap is 38 2024-09-13T14:20:32.799 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 38 2024-09-13T14:20:32.799 INFO:tasks.rados.rados.0.smithi026.stdout:443: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:20:32.799 INFO:tasks.rados.rados.0.smithi026.stdout:449: seq_num 161 ranges {0=303104} 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:449: writing smithi02683976-9 from 0 to 303104 tid 1 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:450: rollback oid 8 current snap is 38 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 33 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:443: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 190 (ObjNum 134 snap 30 seq_num 134) dirty exists 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:443: done (6 left) 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 38 2024-09-13T14:20:32.801 INFO:tasks.rados.rados.0.smithi026.stdout:451: seq_num 162 ranges {827392=319488} 2024-09-13T14:20:32.807 INFO:tasks.rados.rados.0.smithi026.stdout:451: writing smithi02683976-50 from 827392 to 1146880 tid 1 2024-09-13T14:20:32.807 INFO:tasks.rados.rados.0.smithi026.stdout:452: copy_from oid 17 from oid 41 current snap is 38 2024-09-13T14:20:32.808 INFO:tasks.rados.rados.0.smithi026.stdout:442: expect (ObjNum 142 snap 30 seq_num 142) 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:449: finishing write tid 1 to smithi02683976-9 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:449: finishing write tid 2 to smithi02683976-9 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:450: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:449: finishing write tid 3 to smithi02683976-9 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 192 (ObjNum 161 snap 38 seq_num 161) dirty exists 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:449: left oid 9 (ObjNum 161 snap 38 seq_num 161) 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:450: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:442: done (7 left) 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:449: done (6 left) 2024-09-13T14:20:32.878 INFO:tasks.rados.rados.0.smithi026.stdout:453: read oid 20 snap -1 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:453: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 38 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:450: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 36 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:441: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:20:32.879 INFO:tasks.rados.rados.0.smithi026.stdout:445: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:451: finishing write tid 1 to smithi02683976-50 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:451: finishing write tid 2 to smithi02683976-50 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:451: finishing write tid 3 to smithi02683976-50 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 195 (ObjNum 162 snap 38 seq_num 162) dirty exists 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:451: left oid 50 (ObjNum 162 snap 38 seq_num 162) 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:441: finishing copy_from to smithi02683976-15 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 25 (ObjNum 65 snap 5 seq_num 65) dirty exists 2024-09-13T14:20:32.934 INFO:tasks.rados.rados.0.smithi026.stdout:452: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:20:32.935 INFO:tasks.rados.rados.0.smithi026.stdout:452: finishing copy_from to smithi02683976-17 2024-09-13T14:20:32.935 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 85 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-13T14:20:32.935 INFO:tasks.rados.rados.0.smithi026.stdout:448: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:20:32.935 INFO:tasks.rados.rados.0.smithi026.stdout:454: seq_num 163 ranges {90112=335872} 2024-09-13T14:20:32.936 INFO:tasks.rados.rados.0.smithi026.stdout:454: writing smithi02683976-25 from 90112 to 425984 tid 1 2024-09-13T14:20:32.936 INFO:tasks.rados.rados.0.smithi026.stdout:441: done (7 left) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:445: done (6 left) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:450: done (5 left) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:451: done (4 left) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:452: done (3 left) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:455: read oid 47 snap -1 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:455: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 38 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:448: finishing copy_from to smithi02683976-33 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 100 (ObjNum 133 snap 30 seq_num 133) dirty exists 2024-09-13T14:20:32.937 INFO:tasks.rados.rados.0.smithi026.stdout:456: seq_num 164 ranges {4431872=352256} 2024-09-13T14:20:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:32 smithi145 ceph-mon[31057]: pgmap v212: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:32 smithi145 ceph-mon[31057]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:20:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:32 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:32.956 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[33108]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:20:32.962 INFO:tasks.rados.rados.0.smithi026.stdout:456: writing smithi02683976-30 from 4431872 to 4784128 tid 1 2024-09-13T14:20:32.962 INFO:tasks.rados.rados.0.smithi026.stdout:448: done (4 left) 2024-09-13T14:20:32.963 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 38 2024-09-13T14:20:32.963 INFO:tasks.rados.rados.0.smithi026.stdout:457: seq_num 165 ranges {1499136=368640} 2024-09-13T14:20:32.972 INFO:tasks.rados.rados.0.smithi026.stdout:457: writing smithi02683976-39 from 1499136 to 1867776 tid 1 2024-09-13T14:20:32.973 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 38 2024-09-13T14:20:32.973 INFO:tasks.rados.rados.0.smithi026.stdout:458: seq_num 166 ranges {1761280=385024} 2024-09-13T14:20:32.984 INFO:tasks.rados.rados.0.smithi026.stdout:458: writing smithi02683976-15 from 1761280 to 2146304 tid 1 2024-09-13T14:20:32.984 INFO:tasks.rados.rados.0.smithi026.stdout:459: read oid 26 snap -1 2024-09-13T14:20:32.984 INFO:tasks.rados.rados.0.smithi026.stdout:459: expect deleted 2024-09-13T14:20:32.985 INFO:tasks.rados.rados.0.smithi026.stdout:460: snap_remove snap 33 2024-09-13T14:20:33.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[39182]: pgmap v212: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:33.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[39182]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:20:33.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[27770]: pgmap v212: 121 pgs: 121 active+clean; 93 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 24 KiB/s wr, 17 op/s 2024-09-13T14:20:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[27770]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:20:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:32 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:33.799 INFO:tasks.rados.rados.0.smithi026.stdout:460: done (7 left) 2024-09-13T14:20:33.799 INFO:tasks.rados.rados.0.smithi026.stdout:461: rollback oid 36 current snap is 38 2024-09-13T14:20:33.799 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 36 2024-09-13T14:20:33.800 INFO:tasks.rados.rados.0.smithi026.stdout:462: delete oid 44 current snap is 38 2024-09-13T14:20:33.800 INFO:tasks.rados.rados.0.smithi026.stdout:453: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:20:33.802 INFO:tasks.rados.rados.0.smithi026.stdout:455: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:454: finishing write tid 1 to smithi02683976-25 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:454: finishing write tid 2 to smithi02683976-25 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:454: finishing write tid 3 to smithi02683976-25 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 134 (ObjNum 163 snap 38 seq_num 163) dirty exists 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:454: left oid 25 (ObjNum 163 snap 38 seq_num 163) 2024-09-13T14:20:33.805 INFO:tasks.rados.rados.0.smithi026.stdout:456: finishing write tid 1 to smithi02683976-30 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:456: finishing write tid 2 to smithi02683976-30 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:456: finishing write tid 3 to smithi02683976-30 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 119 (ObjNum 164 snap 38 seq_num 164) dirty exists 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:456: left oid 30 (ObjNum 164 snap 38 seq_num 164) 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:457: finishing write tid 1 to smithi02683976-39 2024-09-13T14:20:33.806 INFO:tasks.rados.rados.0.smithi026.stdout:457: finishing write tid 2 to smithi02683976-39 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:457: finishing write tid 3 to smithi02683976-39 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 137 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:457: left oid 39 (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:458: finishing write tid 1 to smithi02683976-15 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:458: finishing write tid 2 to smithi02683976-15 2024-09-13T14:20:33.807 INFO:tasks.rados.rados.0.smithi026.stdout:458: finishing write tid 3 to smithi02683976-15 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 121 (ObjNum 166 snap 38 seq_num 166) dirty exists 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:458: left oid 15 (ObjNum 166 snap 38 seq_num 166) 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:453: done (8 left) 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:454: done (7 left) 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:455: done (6 left) 2024-09-13T14:20:33.808 INFO:tasks.rados.rados.0.smithi026.stdout:456: done (5 left) 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:457: done (4 left) 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:458: done (3 left) 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:459: done (2 left) 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:462: done (1 left) 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 38 2024-09-13T14:20:33.809 INFO:tasks.rados.rados.0.smithi026.stdout:463: seq_num 168 ranges {2719744=417792} 2024-09-13T14:20:33.824 INFO:tasks.rados.rados.0.smithi026.stdout:463: writing smithi02683976-4 from 2719744 to 3137536 tid 1 2024-09-13T14:20:33.825 INFO:tasks.rados.rados.0.smithi026.stdout:464: snap_remove snap 37 2024-09-13T14:20:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:33 smithi026 ceph-mon[27770]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:20:34.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:33 smithi026 ceph-mon[39182]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:20:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:33 smithi145 ceph-mon[31057]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:20:34.800 INFO:tasks.rados.rados.0.smithi026.stdout:464: done (2 left) 2024-09-13T14:20:34.800 INFO:tasks.rados.rados.0.smithi026.stdout:465: copy_from oid 13 from oid 43 current snap is 38 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:461: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:461: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:461: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 252 (ObjNum 152 snap 36 seq_num 152) dirty exists 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:463: finishing write tid 1 to smithi02683976-4 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:463: finishing write tid 2 to smithi02683976-4 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:463: finishing write tid 3 to smithi02683976-4 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 126 (ObjNum 168 snap 38 seq_num 168) dirty exists 2024-09-13T14:20:34.801 INFO:tasks.rados.rados.0.smithi026.stdout:463: left oid 4 (ObjNum 168 snap 38 seq_num 168) 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:461: done (2 left) 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:463: done (1 left) 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:466: setattr oid 27 current snap is 38 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:467: rollback oid 3 current snap is 38 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 31 2024-09-13T14:20:34.802 INFO:tasks.rados.rados.0.smithi026.stdout:468: setattr oid 40 current snap is 38 2024-09-13T14:20:34.803 INFO:tasks.rados.rados.0.smithi026.stdout:469: snap_create 2024-09-13T14:20:34.811 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 129 (ObjNum 65 snap 5 seq_num 65) dirty exists 2024-09-13T14:20:34.813 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 263 (ObjNum 81 snap 0 seq_num 1953066355) dirty exists 2024-09-13T14:20:34.813 INFO:tasks.rados.rados.0.smithi026.stdout:467: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:20:34.813 INFO:tasks.rados.rados.0.smithi026.stdout:467: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:20:34.816 INFO:tasks.rados.rados.0.smithi026.stdout:467: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:20:34.816 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 267 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:34.820 INFO:tasks.rados.rados.0.smithi026.stdout:465: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:20:34.822 INFO:tasks.rados.rados.0.smithi026.stdout:465: finishing copy_from to smithi02683976-13 2024-09-13T14:20:34.822 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 187 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-13T14:20:34.879 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: pgmap v215: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:34.879 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:20:34.879 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:34.880 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:34.880 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:34.880 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:20:34.880 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:34 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:35.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: pgmap v215: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:35.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: pgmap v215: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:35.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:20:35.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:20:35.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:34 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:35.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:35] "GET /metrics HTTP/1.1" 200 240126 "" "Prometheus/2.43.0" 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:465: done (4 left) 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:466: done (3 left) 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:467: done (2 left) 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:468: done (1 left) 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:469: done (0 left) 2024-09-13T14:20:35.811 INFO:tasks.rados.rados.0.smithi026.stdout:470: snap_create 2024-09-13T14:20:35.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:35 smithi145 systemd[1]: Stopping Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:20:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:35 smithi145 ceph-mon[31057]: Upgrade: Updating mgr.x 2024-09-13T14:20:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:35 smithi145 ceph-mon[31057]: Deploying daemon mgr.x on smithi145 2024-09-13T14:20:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:35 smithi145 ceph-mon[31057]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:20:36.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[39182]: Upgrade: Updating mgr.x 2024-09-13T14:20:36.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[39182]: Deploying daemon mgr.x on smithi145 2024-09-13T14:20:36.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[39182]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:20:36.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[27770]: Upgrade: Updating mgr.x 2024-09-13T14:20:36.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[27770]: Deploying daemon mgr.x on smithi145 2024-09-13T14:20:36.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:35 smithi026 ceph-mon[27770]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:20:36.430 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 podman[66599]: 2024-09-13 14:20:36.135139815 +0000 UTC m=+0.132007008 container died 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:20:36.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 podman[66599]: 2024-09-13 14:20:36.528489769 +0000 UTC m=+0.525356958 container cleanup 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD) 2024-09-13T14:20:36.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 bash[66599]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:20:36.812 INFO:tasks.rados.rados.0.smithi026.stdout:470: done (0 left) 2024-09-13T14:20:36.812 INFO:tasks.rados.rados.0.smithi026.stdout:471: setattr oid 4 current snap is 40 2024-09-13T14:20:36.813 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 40 2024-09-13T14:20:36.813 INFO:tasks.rados.rados.0.smithi026.stdout:472: seq_num 169 ranges {1908736=434176} 2024-09-13T14:20:36.826 INFO:tasks.rados.rados.0.smithi026.stdout:472: writing smithi02683976-14 from 1908736 to 2342912 tid 1 2024-09-13T14:20:36.826 INFO:tasks.rados.rados.0.smithi026.stdout:473: read oid 16 snap 31 2024-09-13T14:20:36.826 INFO:tasks.rados.rados.0.smithi026.stdout:473: expect (ObjNum 109 snap 24 seq_num 109) 2024-09-13T14:20:36.827 INFO:tasks.rados.rados.0.smithi026.stdout:474: rmattr oid 10 current snap is 40 2024-09-13T14:20:36.827 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 134 (ObjNum 168 snap 38 seq_num 168) dirty exists 2024-09-13T14:20:36.827 INFO:tasks.rados.rados.0.smithi026.stdout:471: done (3 left) 2024-09-13T14:20:36.827 INFO:tasks.rados.rados.0.smithi026.stdout:475: copy_from oid 41 from oid 18 current snap is 40 2024-09-13T14:20:36.827 INFO:tasks.rados.rados.0.smithi026.stdout:476: delete oid 33 current snap is 40 2024-09-13T14:20:36.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 131 (ObjNum 133 snap 30 seq_num 133) dirty exists 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:474: done (4 left) 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:476: done (3 left) 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:477: rmattr oid 28 current snap is 40 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:477: done (3 left) 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:478: rollback oid 13 current snap is 40 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 34 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:479: copy_from oid 23 from oid 3 current snap is 40 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:480: read oid 6 snap -1 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:480: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:481: rollback oid 5 current snap is 40 2024-09-13T14:20:36.834 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 5 to 36 2024-09-13T14:20:36.835 INFO:tasks.rados.rados.0.smithi026.stdout:482: setattr oid 21 current snap is 40 2024-09-13T14:20:36.835 INFO:tasks.rados.rados.0.smithi026.stdout:472: finishing write tid 1 to smithi02683976-14 2024-09-13T14:20:36.835 INFO:tasks.rados.rados.0.smithi026.stdout:472: finishing write tid 2 to smithi02683976-14 2024-09-13T14:20:36.835 INFO:tasks.rados.rados.0.smithi026.stdout:483: snap_create 2024-09-13T14:20:36.838 INFO:tasks.rados.rados.0.smithi026.stdout:481: finishing rollback tid 0 to smithi02683976-5 2024-09-13T14:20:36.838 INFO:tasks.rados.rados.0.smithi026.stdout:472: finishing write tid 3 to smithi02683976-14 2024-09-13T14:20:36.839 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 201 (ObjNum 169 snap 40 seq_num 169) dirty exists 2024-09-13T14:20:36.839 INFO:tasks.rados.rados.0.smithi026.stdout:472: left oid 14 (ObjNum 169 snap 40 seq_num 169) 2024-09-13T14:20:36.841 INFO:tasks.rados.rados.0.smithi026.stdout:481: finishing rollback tid 1 to smithi02683976-5 2024-09-13T14:20:36.841 INFO:tasks.rados.rados.0.smithi026.stdout:481: finishing rollback tid 2 to smithi02683976-5 2024-09-13T14:20:36.841 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 277 (ObjNum 117 snap 25 seq_num 117) dirty exists 2024-09-13T14:20:36.842 INFO:tasks.rados.rados.0.smithi026.stdout:479: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:20:36.844 INFO:tasks.rados.rados.0.smithi026.stdout:479: finishing copy_from to smithi02683976-23 2024-09-13T14:20:36.844 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 267 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:36.851 INFO:tasks.rados.rados.0.smithi026.stdout:473: expect (ObjNum 109 snap 24 seq_num 109) 2024-09-13T14:20:37.024 INFO:tasks.rados.rados.0.smithi026.stdout:478: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:20:37.024 INFO:tasks.rados.rados.0.smithi026.stdout:478: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:20:37.024 INFO:tasks.rados.rados.0.smithi026.stdout:475: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:20:37.024 INFO:tasks.rados.rados.0.smithi026.stdout:478: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:20:37.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 217 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:37.025 INFO:tasks.rados.rados.0.smithi026.stdout:475: finishing copy_from to smithi02683976-41 2024-09-13T14:20:37.025 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 135 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:20:37.025 INFO:tasks.rados.rados.0.smithi026.stdout:480: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-13T14:20:37.154 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 209 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:36 smithi026 ceph-mon[39182]: pgmap v218: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:36 smithi026 ceph-mon[39182]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:20:37.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:36 smithi026 ceph-mon[27770]: pgmap v218: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:37.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:36 smithi026 ceph-mon[27770]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:36 smithi145 ceph-mon[31057]: pgmap v218: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:36 smithi145 ceph-mon[31057]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 podman[66616]: 2024-09-13 14:20:36.788250212 +0000 UTC m=+0.644444534 container remove 7926da3ac6f22e9b5dc57b7d7a843784654cd6e0f8ec8ab02f674b2a07969511 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2) 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:20:37.200 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 systemd[1]: Stopped Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:20:37.201 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:36 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Consumed 10.238s CPU time. 2024-09-13T14:20:37.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 systemd[1]: Starting Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:20:37.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 podman[66763]: 2024-09-13 14:20:37.316145792 +0000 UTC m=+0.016050062 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:20:37.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 podman[66763]: 2024-09-13 14:20:37.422439649 +0000 UTC m=+0.122343908 container create a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:472: done (8 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:473: done (7 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:475: done (6 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:478: done (5 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:479: done (4 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:480: done (3 left) 2024-09-13T14:20:37.823 INFO:tasks.rados.rados.0.smithi026.stdout:481: done (2 left) 2024-09-13T14:20:37.824 INFO:tasks.rados.rados.0.smithi026.stdout:482: done (1 left) 2024-09-13T14:20:37.824 INFO:tasks.rados.rados.0.smithi026.stdout:483: done (0 left) 2024-09-13T14:20:37.824 INFO:tasks.rados.rados.0.smithi026.stdout:484: rmattr oid 4 current snap is 41 2024-09-13T14:20:37.824 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 41 2024-09-13T14:20:37.824 INFO:tasks.rados.rados.0.smithi026.stdout:485: seq_num 171 ranges {0=466944} 2024-09-13T14:20:37.825 INFO:tasks.rados.rados.0.smithi026.stdout:485: writing smithi02683976-47 from 0 to 466944 tid 1 2024-09-13T14:20:37.825 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 41 2024-09-13T14:20:37.826 INFO:tasks.rados.rados.0.smithi026.stdout:486: seq_num 172 ranges {1761280=483328} 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:486: writing smithi02683976-40 from 1761280 to 2244608 tid 1 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:487: read oid 38 snap 34 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:487: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:488: delete oid 16 current snap is 41 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 136 (ObjNum 168 snap 38 seq_num 168) dirty exists 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:485: finishing write tid 1 to smithi02683976-47 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:485: finishing write tid 2 to smithi02683976-47 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:485: finishing write tid 3 to smithi02683976-47 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 58 (ObjNum 171 snap 41 seq_num 171) dirty exists 2024-09-13T14:20:37.839 INFO:tasks.rados.rados.0.smithi026.stdout:485: left oid 47 (ObjNum 171 snap 41 seq_num 171) 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:484: done (4 left) 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:485: done (3 left) 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:488: done (2 left) 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 41 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:486: finishing write tid 1 to smithi02683976-40 2024-09-13T14:20:37.844 INFO:tasks.rados.rados.0.smithi026.stdout:489: seq_num 174 ranges {3948544=516096} 2024-09-13T14:20:37.869 INFO:tasks.rados.rados.0.smithi026.stdout:489: writing smithi02683976-6 from 3948544 to 4464640 tid 1 2024-09-13T14:20:37.869 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 41 2024-09-13T14:20:37.870 INFO:tasks.rados.rados.0.smithi026.stdout:490: seq_num 175 ranges {2342912=540672} 2024-09-13T14:20:37.885 INFO:tasks.rados.rados.0.smithi026.stdout:490: writing smithi02683976-14 from 2342912 to 2883584 tid 1 2024-09-13T14:20:37.885 INFO:tasks.rados.rados.0.smithi026.stdout:491: rollback oid 17 current snap is 41 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 34 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:486: finishing write tid 2 to smithi02683976-40 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:486: finishing write tid 3 to smithi02683976-40 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 134 (ObjNum 172 snap 41 seq_num 172) dirty exists 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:486: left oid 40 (ObjNum 172 snap 41 seq_num 172) 2024-09-13T14:20:37.886 INFO:tasks.rados.rados.0.smithi026.stdout:487: expect (ObjNum 67 snap 6 seq_num 67) 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:489: finishing write tid 1 to smithi02683976-6 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:489: finishing write tid 2 to smithi02683976-6 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:489: finishing write tid 3 to smithi02683976-6 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 212 (ObjNum 174 snap 41 seq_num 174) dirty exists 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:489: left oid 6 (ObjNum 174 snap 41 seq_num 174) 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:491: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:486: done (4 left) 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:487: done (3 left) 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:489: done (2 left) 2024-09-13T14:20:37.948 INFO:tasks.rados.rados.0.smithi026.stdout:492: read oid 48 snap -1 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:492: expect deleted 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:493: rollback oid 41 current snap is 41 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 41 to 30 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:491: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:490: finishing write tid 1 to smithi02683976-14 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:491: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 149 (ObjNum 88 snap 14 seq_num 88) dirty exists 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:490: finishing write tid 2 to smithi02683976-14 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:490: finishing write tid 3 to smithi02683976-14 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 269 (ObjNum 175 snap 41 seq_num 175) dirty exists 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:490: left oid 14 (ObjNum 175 snap 41 seq_num 175) 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:490: done (3 left) 2024-09-13T14:20:37.949 INFO:tasks.rados.rados.0.smithi026.stdout:491: done (2 left) 2024-09-13T14:20:37.950 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 41 2024-09-13T14:20:37.950 INFO:tasks.rados.rados.0.smithi026.stdout:494: seq_num 176 ranges {4177920=557056} 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:37 smithi145 ceph-mon[31057]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:37 smithi145 ceph-mon[31057]: pgmap v221: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 podman[66763]: 2024-09-13 14:20:37.684177428 +0000 UTC m=+0.384081691 container init a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 podman[66763]: 2024-09-13 14:20:37.68943136 +0000 UTC m=+0.389335616 container start a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 ceph-mgr[66802]: -- 172.21.15.145:0/1571493535 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5643a20d30e0 con 0x5643a20b1400 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 bash[66763]: a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e 2024-09-13T14:20:37.955 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:37 smithi145 systemd[1]: Started Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:20:37.973 INFO:tasks.rados.rados.0.smithi026.stdout:494: writing smithi02683976-13 from 4177920 to 4734976 tid 1 2024-09-13T14:20:37.973 INFO:tasks.rados.rados.0.smithi026.stdout:495: rollback oid 46 current snap is 41 2024-09-13T14:20:37.973 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 31 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:493: finishing rollback tid 0 to smithi02683976-41 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:493: finishing rollback tid 1 to smithi02683976-41 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:493: finishing rollback tid 2 to smithi02683976-41 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 153 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:492: done (3 left) 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:493: done (2 left) 2024-09-13T14:20:37.974 INFO:tasks.rados.rados.0.smithi026.stdout:496: snap_remove snap 39 2024-09-13T14:20:38.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:37 smithi026 ceph-mon[39182]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:20:38.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:37 smithi026 ceph-mon[39182]: pgmap v221: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:38.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:37 smithi026 ceph-mon[27770]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:20:38.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:37 smithi026 ceph-mon[27770]: pgmap v221: 121 pgs: 121 active+clean; 105 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:38.736 INFO:tasks.rados.rados.0.smithi026.stdout:496: done (2 left) 2024-09-13T14:20:38.737 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 41 2024-09-13T14:20:38.737 INFO:tasks.rados.rados.0.smithi026.stdout:495: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:20:38.737 INFO:tasks.rados.rados.0.smithi026.stdout:497: seq_num 177 ranges {0=573440} 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:497: writing smithi02683976-44 from 0 to 573440 tid 1 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:498: delete oid 48 current snap is 41 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:494: finishing write tid 1 to smithi02683976-13 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:495: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:494: finishing write tid 2 to smithi02683976-13 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:495: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 154 (ObjNum 127 snap 29 seq_num 127) dirty exists 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:494: finishing write tid 3 to smithi02683976-13 2024-09-13T14:20:38.740 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 222 (ObjNum 176 snap 41 seq_num 176) dirty exists 2024-09-13T14:20:38.741 INFO:tasks.rados.rados.0.smithi026.stdout:494: left oid 13 (ObjNum 176 snap 41 seq_num 176) 2024-09-13T14:20:38.745 INFO:tasks.rados.rados.0.smithi026.stdout:497: finishing write tid 1 to smithi02683976-44 2024-09-13T14:20:38.745 INFO:tasks.rados.rados.0.smithi026.stdout:497: finishing write tid 2 to smithi02683976-44 2024-09-13T14:20:38.746 INFO:tasks.rados.rados.0.smithi026.stdout:494: done (3 left) 2024-09-13T14:20:38.746 INFO:tasks.rados.rados.0.smithi026.stdout:495: done (2 left) 2024-09-13T14:20:38.746 INFO:tasks.rados.rados.0.smithi026.stdout:498: done (1 left) 2024-09-13T14:20:38.746 INFO:tasks.rados.rados.0.smithi026.stdout:499: snap_create 2024-09-13T14:20:38.747 INFO:tasks.rados.rados.0.smithi026.stdout:497: finishing write tid 3 to smithi02683976-44 2024-09-13T14:20:38.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 156 (ObjNum 177 snap 41 seq_num 177) dirty exists 2024-09-13T14:20:38.747 INFO:tasks.rados.rados.0.smithi026.stdout:497: left oid 44 (ObjNum 177 snap 41 seq_num 177) 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[39182]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:20:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:39.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:38 smithi026 ceph-mon[27770]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:20:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:20:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:38 smithi145 ceph-mon[31057]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:20:39.738 INFO:tasks.rados.rados.0.smithi026.stdout:497: done (1 left) 2024-09-13T14:20:39.738 INFO:tasks.rados.rados.0.smithi026.stdout:499: done (0 left) 2024-09-13T14:20:39.739 INFO:tasks.rados.rados.0.smithi026.stdout:500: snap_create 2024-09-13T14:20:40.740 INFO:tasks.rados.rados.0.smithi026.stdout:500: done (0 left) 2024-09-13T14:20:40.740 INFO:tasks.rados.rados.0.smithi026.stdout:501: snap_remove snap 41 2024-09-13T14:20:41.039 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:40 smithi145 ceph-mon[31057]: pgmap v224: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:41.039 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:40 smithi145 ceph-mon[31057]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:20:41.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:40 smithi026 ceph-mon[27770]: pgmap v224: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:41.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:40 smithi026 ceph-mon[27770]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:20:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:40 smithi026 ceph-mon[39182]: pgmap v224: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:40 smithi026 ceph-mon[39182]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:20:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:501: done (0 left) 2024-09-13T14:20:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:502: read oid 21 snap -1 2024-09-13T14:20:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:502: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:503: snap_remove snap 40 2024-09-13T14:20:42.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:41 smithi026 ceph-mon[27770]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:20:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:41 smithi026 ceph-mon[39182]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:20:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:41 smithi145 ceph-mon[31057]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:20:42.751 INFO:tasks.rados.rados.0.smithi026.stdout:503: done (1 left) 2024-09-13T14:20:42.752 INFO:tasks.rados.rados.0.smithi026.stdout:504: snap_remove snap 38 2024-09-13T14:20:43.040 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:42 smithi145 ceph-mon[31057]: pgmap v227: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:43.040 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:42 smithi145 ceph-mon[31057]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:20:43.040 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:42 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.040 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:42 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[27770]: pgmap v227: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[27770]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[39182]: pgmap v227: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 37 op/s 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[39182]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:42 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.304 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:43] ENGINE Bus STARTING 2024-09-13T14:20:43.305 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: CherryPy Checker: 2024-09-13T14:20:43.305 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: The Application mounted at '' has an empty config. 2024-09-13T14:20:43.305 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:43.305 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:43] ENGINE Serving on http://:::9283 2024-09-13T14:20:43.305 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:43] ENGINE Bus STARTED 2024-09-13T14:20:43.763 INFO:tasks.rados.rados.0.smithi026.stdout:502: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:43.769 INFO:tasks.rados.rados.0.smithi026.stdout:502: done (1 left) 2024-09-13T14:20:43.769 INFO:tasks.rados.rados.0.smithi026.stdout:504: done (0 left) 2024-09-13T14:20:43.769 INFO:tasks.rados.rados.0.smithi026.stdout:505: copy_from oid 40 from oid 18 current snap is 43 2024-09-13T14:20:43.770 INFO:tasks.rados.rados.0.smithi026.stdout:506: rollback oid 8 current snap is 43 2024-09-13T14:20:43.770 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 36 2024-09-13T14:20:43.770 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 43 2024-09-13T14:20:43.770 INFO:tasks.rados.rados.0.smithi026.stdout:507: seq_num 179 ranges {2285568=606208} 2024-09-13T14:20:43.786 INFO:tasks.rados.rados.0.smithi026.stdout:507: writing smithi02683976-43 from 2285568 to 2891776 tid 1 2024-09-13T14:20:43.786 INFO:tasks.rados.rados.0.smithi026.stdout:508: snap_create 2024-09-13T14:20:43.787 INFO:tasks.rados.rados.0.smithi026.stdout:506: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:20:43.787 INFO:tasks.rados.rados.0.smithi026.stdout:506: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:20:43.787 INFO:tasks.rados.rados.0.smithi026.stdout:506: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:20:43.787 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 44 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-13T14:20:43.792 INFO:tasks.rados.rados.0.smithi026.stdout:507: finishing write tid 1 to smithi02683976-43 2024-09-13T14:20:43.792 INFO:tasks.rados.rados.0.smithi026.stdout:505: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:20:43.793 INFO:tasks.rados.rados.0.smithi026.stdout:507: finishing write tid 2 to smithi02683976-43 2024-09-13T14:20:43.794 INFO:tasks.rados.rados.0.smithi026.stdout:505: finishing copy_from to smithi02683976-40 2024-09-13T14:20:43.794 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 135 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:20:43.794 INFO:tasks.rados.rados.0.smithi026.stdout:507: finishing write tid 3 to smithi02683976-43 2024-09-13T14:20:43.795 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 237 (ObjNum 179 snap 43 seq_num 179) dirty exists 2024-09-13T14:20:43.795 INFO:tasks.rados.rados.0.smithi026.stdout:507: left oid 43 (ObjNum 179 snap 43 seq_num 179) 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: Standby manager daemon x restarted 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:20:43.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:43 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: Standby manager daemon x restarted 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:20:44.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: Standby manager daemon x restarted 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: Standby manager daemon x started 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:20:44.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:44.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:20:44.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:43 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:505: done (3 left) 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:506: done (2 left) 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:507: done (1 left) 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:508: done (0 left) 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 44 2024-09-13T14:20:44.765 INFO:tasks.rados.rados.0.smithi026.stdout:509: seq_num 180 ranges {4235264=622592} 2024-09-13T14:20:44.791 INFO:tasks.rados.rados.0.smithi026.stdout:509: writing smithi02683976-17 from 4235264 to 4857856 tid 1 2024-09-13T14:20:44.791 INFO:tasks.rados.rados.0.smithi026.stdout:510: copy_from oid 28 from oid 42 current snap is 44 2024-09-13T14:20:44.791 INFO:tasks.rados.rados.0.smithi026.stdout:511: rollback oid 15 current snap is 44 2024-09-13T14:20:44.792 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 42 2024-09-13T14:20:44.792 INFO:tasks.rados.rados.0.smithi026.stdout:512: rollback oid 33 current snap is 44 2024-09-13T14:20:44.792 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 43 2024-09-13T14:20:44.792 INFO:tasks.rados.rados.0.smithi026.stdout:513: delete oid 46 current snap is 44 2024-09-13T14:20:44.794 INFO:tasks.rados.rados.0.smithi026.stdout:511: finishing rollback tid 0 to smithi02683976-15 2024-09-13T14:20:44.796 INFO:tasks.rados.rados.0.smithi026.stdout:511: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:20:44.796 INFO:tasks.rados.rados.0.smithi026.stdout:511: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:20:44.796 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 146 (ObjNum 166 snap 38 seq_num 166) dirty exists 2024-09-13T14:20:44.799 INFO:tasks.rados.rados.0.smithi026.stdout:509: finishing write tid 1 to smithi02683976-17 2024-09-13T14:20:44.799 INFO:tasks.rados.rados.0.smithi026.stdout:509: finishing write tid 2 to smithi02683976-17 2024-09-13T14:20:44.799 INFO:tasks.rados.rados.0.smithi026.stdout:512: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:20:44.799 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 273 (ObjNum 129 snap 0 seq_num 519164208) dirty dne 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:511: done (4 left) 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:512: done (3 left) 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:513: done (2 left) 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:514: read oid 21 snap 30 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:514: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:44.800 INFO:tasks.rados.rados.0.smithi026.stdout:515: read oid 39 snap -1 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:515: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:516: copy_from oid 46 from oid 19 current snap is 44 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:517: delete oid 35 current snap is 44 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:509: finishing write tid 3 to smithi02683976-17 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 168 (ObjNum 180 snap 44 seq_num 180) dirty exists 2024-09-13T14:20:44.801 INFO:tasks.rados.rados.0.smithi026.stdout:509: left oid 17 (ObjNum 180 snap 44 seq_num 180) 2024-09-13T14:20:44.802 INFO:tasks.rados.rados.0.smithi026.stdout:514: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-13T14:20:44.806 INFO:tasks.rados.rados.0.smithi026.stdout:509: done (5 left) 2024-09-13T14:20:44.806 INFO:tasks.rados.rados.0.smithi026.stdout:514: done (4 left) 2024-09-13T14:20:44.806 INFO:tasks.rados.rados.0.smithi026.stdout:517: done (3 left) 2024-09-13T14:20:44.806 INFO:tasks.rados.rados.0.smithi026.stdout:518: read oid 2 snap -1 2024-09-13T14:20:44.806 INFO:tasks.rados.rados.0.smithi026.stdout:518: expect deleted 2024-09-13T14:20:44.807 INFO:tasks.rados.rados.0.smithi026.stdout:519: delete oid 10 current snap is 44 2024-09-13T14:20:44.808 INFO:tasks.rados.rados.0.smithi026.stdout:515: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:20:44.893 INFO:tasks.rados.rados.0.smithi026.stdout:516: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:510: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:516: finishing copy_from to smithi02683976-46 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 117 (ObjNum 120 snap 27 seq_num 120) dirty exists 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:510: finishing copy_from to smithi02683976-28 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 54 (ObjNum 116 snap 24 seq_num 116) dirty exists 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:510: done (4 left) 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:515: done (3 left) 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:516: done (2 left) 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:518: done (1 left) 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:519: done (0 left) 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:520: copy_from oid 8 from oid 29 current snap is 44 2024-09-13T14:20:44.894 INFO:tasks.rados.rados.0.smithi026.stdout:521: delete oid 10 current snap is 44 2024-09-13T14:20:44.895 INFO:tasks.rados.rados.0.smithi026.stdout:521: done (1 left) 2024-09-13T14:20:44.895 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 44 2024-09-13T14:20:44.895 INFO:tasks.rados.rados.0.smithi026.stdout:522: seq_num 185 ranges {466944=704512} 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:522: writing smithi02683976-47 from 466944 to 1171456 tid 1 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:523: rollback oid 21 current snap is 44 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 31 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:520: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:520: finishing copy_from to smithi02683976-8 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 201 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:520: done (2 left) 2024-09-13T14:20:44.902 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 44 2024-09-13T14:20:44.903 INFO:tasks.rados.rados.0.smithi026.stdout:524: seq_num 186 ranges {2891776=720896} 2024-09-13T14:20:44.921 INFO:tasks.rados.rados.0.smithi026.stdout:524: writing smithi02683976-43 from 2891776 to 3612672 tid 1 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:525: delete oid 5 current snap is 44 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:523: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:522: finishing write tid 1 to smithi02683976-47 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:523: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:522: finishing write tid 2 to smithi02683976-47 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:523: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 226 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:522: finishing write tid 3 to smithi02683976-47 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 68 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-13T14:20:44.922 INFO:tasks.rados.rados.0.smithi026.stdout:522: left oid 47 (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:20:44.923 INFO:tasks.rados.rados.0.smithi026.stdout:522: done (3 left) 2024-09-13T14:20:44.923 INFO:tasks.rados.rados.0.smithi026.stdout:523: done (2 left) 2024-09-13T14:20:44.923 INFO:tasks.rados.rados.0.smithi026.stdout:525: done (1 left) 2024-09-13T14:20:44.923 INFO:tasks.rados.rados.0.smithi026.stdout:526: delete oid 10 current snap is 44 2024-09-13T14:20:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:526: done (1 left) 2024-09-13T14:20:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:527: rmattr oid 29 current snap is 44 2024-09-13T14:20:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 44 2024-09-13T14:20:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:528: seq_num 189 ranges {2883584=770048} 2024-09-13T14:20:44.945 INFO:tasks.rados.rados.0.smithi026.stdout:528: writing smithi02683976-14 from 2883584 to 3653632 tid 1 2024-09-13T14:20:44.945 INFO:tasks.rados.rados.0.smithi026.stdout:529: read oid 26 snap -1 2024-09-13T14:20:44.945 INFO:tasks.rados.rados.0.smithi026.stdout:529: expect deleted 2024-09-13T14:20:44.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 44 2024-09-13T14:20:44.945 INFO:tasks.rados.rados.0.smithi026.stdout:530: seq_num 190 ranges {3252224=786432} 2024-09-13T14:20:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:44 smithi145 ceph-mon[31057]: pgmap v230: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:44 smithi145 ceph-mon[31057]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:20:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:44 smithi145 ceph-mon[31057]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-13T14:20:44.967 INFO:tasks.rados.rados.0.smithi026.stdout:530: writing smithi02683976-40 from 3252224 to 4038656 tid 1 2024-09-13T14:20:44.967 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 44 2024-09-13T14:20:44.967 INFO:tasks.rados.rados.0.smithi026.stdout:531: seq_num 191 ranges {0=802816} 2024-09-13T14:20:44.972 INFO:tasks.rados.rados.0.smithi026.stdout:531: writing smithi02683976-10 from 0 to 802816 tid 1 2024-09-13T14:20:44.972 INFO:tasks.rados.rados.0.smithi026.stdout:532: snap_remove snap 30 2024-09-13T14:20:45.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[27770]: pgmap v230: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:45.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[27770]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:20:45.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[27770]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-13T14:20:45.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[39182]: pgmap v230: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:45.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[39182]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:20:45.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:44 smithi026 ceph-mon[39182]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-13T14:20:45.336 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:45] "GET /metrics HTTP/1.1" 200 240126 "" "Prometheus/2.43.0" 2024-09-13T14:20:45.775 INFO:tasks.rados.rados.0.smithi026.stdout:524: finishing write tid 1 to smithi02683976-43 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:524: finishing write tid 2 to smithi02683976-43 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 242 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:524: finishing write tid 3 to smithi02683976-43 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 240 (ObjNum 186 snap 44 seq_num 186) dirty exists 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:524: left oid 43 (ObjNum 186 snap 44 seq_num 186) 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:528: finishing write tid 1 to smithi02683976-14 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:528: finishing write tid 2 to smithi02683976-14 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:528: finishing write tid 3 to smithi02683976-14 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 271 (ObjNum 189 snap 44 seq_num 189) dirty exists 2024-09-13T14:20:45.821 INFO:tasks.rados.rados.0.smithi026.stdout:528: left oid 14 (ObjNum 189 snap 44 seq_num 189) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:530: finishing write tid 1 to smithi02683976-40 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:530: finishing write tid 2 to smithi02683976-40 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:530: finishing write tid 3 to smithi02683976-40 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 155 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:530: left oid 40 (ObjNum 190 snap 44 seq_num 190) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:531: finishing write tid 1 to smithi02683976-10 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:531: finishing write tid 2 to smithi02683976-10 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:531: finishing write tid 3 to smithi02683976-10 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 157 (ObjNum 191 snap 44 seq_num 191) dirty exists 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:531: left oid 10 (ObjNum 191 snap 44 seq_num 191) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:524: done (6 left) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:527: done (5 left) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:528: done (4 left) 2024-09-13T14:20:45.822 INFO:tasks.rados.rados.0.smithi026.stdout:529: done (3 left) 2024-09-13T14:20:45.823 INFO:tasks.rados.rados.0.smithi026.stdout:530: done (2 left) 2024-09-13T14:20:45.823 INFO:tasks.rados.rados.0.smithi026.stdout:531: done (1 left) 2024-09-13T14:20:45.823 INFO:tasks.rados.rados.0.smithi026.stdout:532: done (0 left) 2024-09-13T14:20:45.823 INFO:tasks.rados.rados.0.smithi026.stdout:533: snap_create 2024-09-13T14:20:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:45 smithi145 ceph-mon[31057]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:20:46.102 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:45 smithi026 ceph-mon[39182]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:20:46.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:45 smithi026 ceph-mon[27770]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:20:46.102 INFO:tasks.rados.rados.0.smithi026.stdout:533: done (0 left) 2024-09-13T14:20:46.103 INFO:tasks.rados.rados.0.smithi026.stdout:534: snap_remove snap 31 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.100+0000 7f58257f2640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setuser ceph since I am not root 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ignoring --setgroup ceph since I am not root 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mgr[28045]: -- 172.21.15.26:0/4106148152 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558dacc00340 con 0x558dacc0cc00 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.249+0000 7f24fbe27ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:20:46.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.307+0000 7f24fbe27ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:20:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE Bus STOPPING 2024-09-13T14:20:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:20:46.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE Bus STOPPED 2024-09-13T14:20:46.652 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.376+0000 7f24fbe27ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:20:46.652 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.505+0000 7f24fbe27ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:20:46.652 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.650+0000 7f24fbe27ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: pgmap v233: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: Activating manager daemon x 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24425 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: mgrmap e24: x(active, starting, since 0.0077059s) 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:20:46.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: Manager daemon x is now available 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: pgmap v233: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:46.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: Activating manager daemon x 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24425 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: mgrmap e24: x(active, starting, since 0.0077059s) 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:20:46.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:20:46.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: Manager daemon x is now available 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:46.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE Bus STARTING 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e941ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e941ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e941ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e941ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e941ad640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e971b3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e971b3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e971b3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e971b3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:46.821+0000 7f3e971b3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE Serving on http://:::9283 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:20:46] ENGINE Bus STARTED 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: pgmap v233: 121 pgs: 121 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 1 op/s 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:47.101 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 172.21.15.26:0/795216309' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:20:47.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: Activating manager daemon x 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24425 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: mgrmap e24: x(active, starting, since 0.0077059s) 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:47.104 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:47.105 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: Manager daemon x is now available 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:47.106 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:534: done (0 left) 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:535: read oid 16 snap 43 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:535: expect deleted 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 45 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:536: seq_num 192 ranges {0=417792,417792=401408} 2024-09-13T14:20:47.107 INFO:tasks.rados.rados.0.smithi026.stdout:536: writing smithi02683976-2 from 0 to 417792 tid 1 2024-09-13T14:20:47.109 INFO:tasks.rados.rados.0.smithi026.stdout:536: writing smithi02683976-2 from 417792 to 819200 tid 2 2024-09-13T14:20:47.109 INFO:tasks.rados.rados.0.smithi026.stdout:537: setattr oid 49 current snap is 45 2024-09-13T14:20:47.110 INFO:tasks.rados.rados.0.smithi026.stdout:535: done (2 left) 2024-09-13T14:20:47.110 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 45 2024-09-13T14:20:47.110 INFO:tasks.rados.rados.0.smithi026.stdout:538: seq_num 193 ranges {0=434176,434176=401408} 2024-09-13T14:20:47.113 INFO:tasks.rados.rados.0.smithi026.stdout:538: writing smithi02683976-33 from 0 to 434176 tid 1 2024-09-13T14:20:47.116 INFO:tasks.rados.rados.0.smithi026.stdout:538: writing smithi02683976-33 from 434176 to 835584 tid 2 2024-09-13T14:20:47.117 INFO:tasks.rados.rados.0.smithi026.stdout:539: read oid 29 snap -1 2024-09-13T14:20:47.117 INFO:tasks.rados.rados.0.smithi026.stdout:539: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:20:47.117 INFO:tasks.rados.rados.0.smithi026.stdout:540: delete oid 38 current snap is 45 2024-09-13T14:20:47.124 INFO:tasks.rados.rados.0.smithi026.stdout:538: finishing write tid 1 to smithi02683976-33 2024-09-13T14:20:47.125 INFO:tasks.rados.rados.0.smithi026.stdout:538: finishing write tid 2 to smithi02683976-33 2024-09-13T14:20:47.125 INFO:tasks.rados.rados.0.smithi026.stdout:538: finishing write tid 3 to smithi02683976-33 2024-09-13T14:20:47.126 INFO:tasks.rados.rados.0.smithi026.stdout:536: finishing write tid 1 to smithi02683976-2 2024-09-13T14:20:47.131 INFO:tasks.rados.rados.0.smithi026.stdout:539: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:20:47.154 INFO:tasks.rados.rados.0.smithi026.stdout:536: finishing write tid 2 to smithi02683976-2 2024-09-13T14:20:47.154 INFO:tasks.rados.rados.0.smithi026.stdout:536: finishing write tid 3 to smithi02683976-2 2024-09-13T14:20:47.154 INFO:tasks.rados.rados.0.smithi026.stdout:536: finishing write tid 4 to smithi02683976-2 2024-09-13T14:20:47.154 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 162 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-13T14:20:47.154 INFO:tasks.rados.rados.0.smithi026.stdout:536: left oid 2 (ObjNum 192 snap 45 seq_num 192) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:538: finishing write tid 4 to smithi02683976-33 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 304 (ObjNum 193 snap 45 seq_num 193) dirty exists 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:538: left oid 33 (ObjNum 193 snap 45 seq_num 193) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 164 (ObjNum 129 snap 0 seq_num 519019480) dirty exists 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:536: done (4 left) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:537: done (3 left) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:538: done (2 left) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:539: done (1 left) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:540: done (0 left) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:541: read oid 28 snap 44 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:541: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:542: read oid 2 snap 34 2024-09-13T14:20:47.155 INFO:tasks.rados.rados.0.smithi026.stdout:542: expect deleted 2024-09-13T14:20:47.156 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 45 2024-09-13T14:20:47.156 INFO:tasks.rados.rados.0.smithi026.stdout:543: seq_num 195 ranges {3653632=475136,4128768=401408} 2024-09-13T14:20:47.176 INFO:tasks.rados.rados.0.smithi026.stdout:543: writing smithi02683976-42 from 3653632 to 4128768 tid 1 2024-09-13T14:20:47.179 INFO:tasks.rados.rados.0.smithi026.stdout:543: writing smithi02683976-42 from 4128768 to 4530176 tid 2 2024-09-13T14:20:47.179 INFO:tasks.rados.rados.0.smithi026.stdout:541: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:20:47.192 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.926+0000 7f24fbe27ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:20:47.192 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:46 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:46.989+0000 7f24fbe27ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:543: finishing write tid 1 to smithi02683976-42 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:543: finishing write tid 2 to smithi02683976-42 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:543: finishing write tid 3 to smithi02683976-42 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:543: finishing write tid 4 to smithi02683976-42 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 226 (ObjNum 195 snap 45 seq_num 195) dirty exists 2024-09-13T14:20:47.307 INFO:tasks.rados.rados.0.smithi026.stdout:543: left oid 42 (ObjNum 195 snap 45 seq_num 195) 2024-09-13T14:20:47.308 INFO:tasks.rados.rados.0.smithi026.stdout:541: done (2 left) 2024-09-13T14:20:47.308 INFO:tasks.rados.rados.0.smithi026.stdout:542: done (1 left) 2024-09-13T14:20:47.308 INFO:tasks.rados.rados.0.smithi026.stdout:543: done (0 left) 2024-09-13T14:20:47.308 INFO:tasks.rados.rados.0.smithi026.stdout:544: snap_create 2024-09-13T14:20:47.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:47.190+0000 7f24fbe27ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: Checking for cert/key for grafana.a 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: Migrating grafana.a cert to cert store 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: Migrating grafana.a key to cert store 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:20:48.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: mgrmap e25: x(active, since 1.01862s) 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: pgmap v4: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: Deploying cephadm binary to smithi145 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[27770]: Deploying cephadm binary to smithi026 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: Checking for cert/key for grafana.a 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: Migrating grafana.a cert to cert store 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: Migrating grafana.a key to cert store 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: mgrmap e25: x(active, since 1.01862s) 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: pgmap v4: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: Deploying cephadm binary to smithi145 2024-09-13T14:20:48.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:47 smithi026 ceph-mon[39182]: Deploying cephadm binary to smithi026 2024-09-13T14:20:48.116 INFO:tasks.rados.rados.0.smithi026.stdout:544: done (0 left) 2024-09-13T14:20:48.116 INFO:tasks.rados.rados.0.smithi026.stdout:545: setattr oid 5 current snap is 46 2024-09-13T14:20:48.118 INFO:tasks.rados.rados.0.smithi026.stdout:546: rollback oid 50 current snap is 46 2024-09-13T14:20:48.118 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 44 2024-09-13T14:20:48.118 INFO:tasks.rados.rados.0.smithi026.stdout:547: setattr oid 47 current snap is 46 2024-09-13T14:20:48.122 INFO:tasks.rados.rados.0.smithi026.stdout:548: rollback oid 10 current snap is 46 2024-09-13T14:20:48.122 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 36 2024-09-13T14:20:48.122 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 46 2024-09-13T14:20:48.122 INFO:tasks.rados.rados.0.smithi026.stdout:549: seq_num 196 ranges {172032=491520,663552=401408} 2024-09-13T14:20:48.126 INFO:tasks.rados.rados.0.smithi026.stdout:549: writing smithi02683976-11 from 172032 to 663552 tid 1 2024-09-13T14:20:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:549: writing smithi02683976-11 from 663552 to 1064960 tid 2 2024-09-13T14:20:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:550: snap_remove snap 34 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: Checking for cert/key for grafana.a 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: Migrating grafana.a cert to cert store 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: Migrating grafana.a key to cert store 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:20:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: mgrmap e25: x(active, since 1.01862s) 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: pgmap v4: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: Deploying cephadm binary to smithi145 2024-09-13T14:20:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:47 smithi145 ceph-mon[31057]: Deploying cephadm binary to smithi026 2024-09-13T14:20:48.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:48 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:48.085+0000 7f24fbe27ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:20:48.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:48 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:48.160+0000 7f24fbe27ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:20:48.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:48.223+0000 7f3e7a6fd640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:20:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:20:48.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: Traceback (most recent call last): 2024-09-13T14:20:48.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:20:48.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: self._connections.run(self.expiration_interval) 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: self._run(expiration_interval) 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: s = self.context.wrap_socket( 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: return self.sslsocket_class._create( 2024-09-13T14:20:48.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:20:48.707 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: self.do_handshake() 2024-09-13T14:20:48.707 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:20:48.707 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: self._sslobj.do_handshake() 2024-09-13T14:20:48.707 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:20:48.707 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:20:48.739 INFO:tasks.rados.rados.0.smithi026.stdout:550: done (5 left) 2024-09-13T14:20:48.740 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 46 2024-09-13T14:20:48.740 INFO:tasks.rados.rados.0.smithi026.stdout:551: seq_num 197 ranges {16384=507904,524288=401408} 2024-09-13T14:20:48.743 INFO:tasks.rados.rados.0.smithi026.stdout:551: writing smithi02683976-23 from 16384 to 524288 tid 1 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:551: writing smithi02683976-23 from 524288 to 925696 tid 2 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:552: rollback oid 20 current snap is 46 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 36 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:546: finishing rollback tid 0 to smithi02683976-50 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 317 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:546: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:546: finishing rollback tid 2 to smithi02683976-50 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 246 (ObjNum 162 snap 38 seq_num 162) dirty exists 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:548: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:545: done (6 left) 2024-09-13T14:20:48.747 INFO:tasks.rados.rados.0.smithi026.stdout:546: done (5 left) 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:553: rmattr oid 2 current snap is 46 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:553: done (5 left) 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:554: read oid 28 snap -1 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:554: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:555: snap_create 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 74 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:548: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:548: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 174 (ObjNum 133 snap 30 seq_num 133) dirty exists 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:549: finishing write tid 1 to smithi02683976-11 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:549: finishing write tid 2 to smithi02683976-11 2024-09-13T14:20:48.748 INFO:tasks.rados.rados.0.smithi026.stdout:549: finishing write tid 3 to smithi02683976-11 2024-09-13T14:20:48.749 INFO:tasks.rados.rados.0.smithi026.stdout:549: finishing write tid 4 to smithi02683976-11 2024-09-13T14:20:48.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 250 (ObjNum 196 snap 46 seq_num 196) dirty exists 2024-09-13T14:20:48.749 INFO:tasks.rados.rados.0.smithi026.stdout:549: left oid 11 (ObjNum 196 snap 46 seq_num 196) 2024-09-13T14:20:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:552: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:20:48.760 INFO:tasks.rados.rados.0.smithi026.stdout:551: finishing write tid 1 to smithi02683976-23 2024-09-13T14:20:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:551: finishing write tid 2 to smithi02683976-23 2024-09-13T14:20:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:551: finishing write tid 3 to smithi02683976-23 2024-09-13T14:20:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:552: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:20:48.764 INFO:tasks.rados.rados.0.smithi026.stdout:552: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:20:48.765 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 266 (ObjNum 129 snap 0 seq_num 519310072) dirty exists 2024-09-13T14:20:48.771 INFO:tasks.rados.rados.0.smithi026.stdout:551: finishing write tid 4 to smithi02683976-23 2024-09-13T14:20:48.772 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 275 (ObjNum 197 snap 46 seq_num 197) dirty exists 2024-09-13T14:20:48.772 INFO:tasks.rados.rados.0.smithi026.stdout:551: left oid 23 (ObjNum 197 snap 46 seq_num 197) 2024-09-13T14:20:48.777 INFO:tasks.rados.rados.0.smithi026.stdout:554: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:20:49.189 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:48 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:48.935+0000 7f24fbe27ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:20:49.189 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.056+0000 7f24fbe27ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:20:49.189 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.187+0000 7f24fbe27ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:20:49.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: [13/Sep/2024:14:20:47] ENGINE Bus STARTING 2024-09-13T14:20:49.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: [13/Sep/2024:14:20:48] ENGINE Serving on http://172.21.15.145:8765 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: pgmap v5: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: [13/Sep/2024:14:20:48] ENGINE Serving on https://172.21.15.145:7150 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: [13/Sep/2024:14:20:48] ENGINE Bus STARTED 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: [13/Sep/2024:14:20:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:20:49.190 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:20:49.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[27770]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: [13/Sep/2024:14:20:47] ENGINE Bus STARTING 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: [13/Sep/2024:14:20:48] ENGINE Serving on http://172.21.15.145:8765 2024-09-13T14:20:49.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: pgmap v5: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: [13/Sep/2024:14:20:48] ENGINE Serving on https://172.21.15.145:7150 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: [13/Sep/2024:14:20:48] ENGINE Bus STARTED 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: [13/Sep/2024:14:20:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:20:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.do_handshake() 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:20:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-mon[39182]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:20:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: [13/Sep/2024:14:20:47] ENGINE Bus STARTING 2024-09-13T14:20:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: [13/Sep/2024:14:20:48] ENGINE Serving on http://172.21.15.145:8765 2024-09-13T14:20:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: pgmap v5: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:20:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: [13/Sep/2024:14:20:48] ENGINE Serving on https://172.21.15.145:7150 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: [13/Sep/2024:14:20:48] ENGINE Bus STARTED 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: [13/Sep/2024:14:20:48] ENGINE Error in HTTPServer.serve 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: Traceback (most recent call last): 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._run(expiration_interval) 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:20:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: s = self.context.wrap_socket( 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: return self.sslsocket_class._create( 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: self.do_handshake() 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._sslobj.do_handshake() 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:20:49.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:49 smithi145 ceph-mon[31057]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:20:49.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.258+0000 7f24fbe27ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:20:49.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.405+0000 7f24fbe27ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:20:49.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.475+0000 7f24fbe27ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:20:49.739 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.536+0000 7f24fbe27ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:20:49.739 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.684+0000 7f24fbe27ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:547: done (6 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:548: done (5 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:549: done (4 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:551: done (3 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:552: done (2 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:554: done (1 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:555: done (0 left) 2024-09-13T14:20:49.753 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 47 2024-09-13T14:20:49.754 INFO:tasks.rados.rados.0.smithi026.stdout:556: seq_num 198 ranges {16384=524288,540672=401408} 2024-09-13T14:20:49.756 INFO:tasks.rados.rados.0.smithi026.stdout:556: writing smithi02683976-3 from 16384 to 540672 tid 1 2024-09-13T14:20:49.758 INFO:tasks.rados.rados.0.smithi026.stdout:556: writing smithi02683976-3 from 540672 to 942080 tid 2 2024-09-13T14:20:49.758 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 47 2024-09-13T14:20:49.758 INFO:tasks.rados.rados.0.smithi026.stdout:557: seq_num 199 ranges {0=540672,540672=401408} 2024-09-13T14:20:49.762 INFO:tasks.rados.rados.0.smithi026.stdout:557: writing smithi02683976-49 from 0 to 540672 tid 1 2024-09-13T14:20:49.765 INFO:tasks.rados.rados.0.smithi026.stdout:557: writing smithi02683976-49 from 540672 to 942080 tid 2 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:558: read oid 32 snap 46 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:558: expect deleted 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:559: rollback oid 50 current snap is 47 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 44 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:556: finishing write tid 1 to smithi02683976-3 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:556: finishing write tid 2 to smithi02683976-3 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:556: finishing write tid 3 to smithi02683976-3 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:560: snap_create 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:556: finishing write tid 4 to smithi02683976-3 2024-09-13T14:20:49.766 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 329 (ObjNum 198 snap 47 seq_num 198) dirty exists 2024-09-13T14:20:49.767 INFO:tasks.rados.rados.0.smithi026.stdout:556: left oid 3 (ObjNum 198 snap 47 seq_num 198) 2024-09-13T14:20:49.767 INFO:tasks.rados.rados.0.smithi026.stdout:557: finishing write tid 1 to smithi02683976-49 2024-09-13T14:20:49.767 INFO:tasks.rados.rados.0.smithi026.stdout:557: finishing write tid 2 to smithi02683976-49 2024-09-13T14:20:49.767 INFO:tasks.rados.rados.0.smithi026.stdout:557: finishing write tid 3 to smithi02683976-49 2024-09-13T14:20:49.769 INFO:tasks.rados.rados.0.smithi026.stdout:559: finishing rollback tid 0 to smithi02683976-50 2024-09-13T14:20:49.769 INFO:tasks.rados.rados.0.smithi026.stdout:557: finishing write tid 4 to smithi02683976-49 2024-09-13T14:20:49.770 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 184 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-13T14:20:49.770 INFO:tasks.rados.rados.0.smithi026.stdout:557: left oid 49 (ObjNum 199 snap 47 seq_num 199) 2024-09-13T14:20:49.770 INFO:tasks.rados.rados.0.smithi026.stdout:559: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:20:49.770 INFO:tasks.rados.rados.0.smithi026.stdout:559: finishing rollback tid 2 to smithi02683976-50 2024-09-13T14:20:49.770 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 264 (ObjNum 162 snap 38 seq_num 162) dirty exists 2024-09-13T14:20:50.039 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.757+0000 7f24fbe27ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:20:50.040 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.829+0000 7f24fbe27ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:20:50.040 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.915+0000 7f24fbe27ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:20:50.040 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:49.975+0000 7f24fbe27ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:20:50.040 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:50.037+0000 7f24fbe27ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:556: done (4 left) 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:557: done (3 left) 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:558: done (2 left) 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:559: done (1 left) 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:560: done (0 left) 2024-09-13T14:20:50.756 INFO:tasks.rados.rados.0.smithi026.stdout:561: read oid 22 snap -1 2024-09-13T14:20:50.757 INFO:tasks.rados.rados.0.smithi026.stdout:561: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:20:50.757 INFO:tasks.rados.rados.0.smithi026.stdout:562: rollback oid 47 current snap is 48 2024-09-13T14:20:50.757 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 46 2024-09-13T14:20:50.757 INFO:tasks.rados.rados.0.smithi026.stdout:563: snap_remove snap 36 2024-09-13T14:20:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[39182]: mgrmap e26: x(active, since 3s) 2024-09-13T14:20:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[39182]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[39182]: pgmap v9: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[27770]: mgrmap e26: x(active, since 3s) 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[27770]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-mon[27770]: pgmap v9: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:50.864+0000 7f24fbe27ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:20:50] ENGINE Bus STARTING 2024-09-13T14:20:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: CherryPy Checker: 2024-09-13T14:20:51.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: The Application mounted at '' has an empty config. 2024-09-13T14:20:51.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: 2024-09-13T14:20:51.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:20:50] ENGINE Serving on http://:::9283 2024-09-13T14:20:51.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:50 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: [13/Sep/2024:14:20:50] ENGINE Bus STARTED 2024-09-13T14:20:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:50 smithi145 ceph-mon[31057]: mgrmap e26: x(active, since 3s) 2024-09-13T14:20:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:50 smithi145 ceph-mon[31057]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:20:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:50 smithi145 ceph-mon[31057]: pgmap v9: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:51.757 INFO:tasks.rados.rados.0.smithi026.stdout:563: done (2 left) 2024-09-13T14:20:51.757 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 48 2024-09-13T14:20:51.758 INFO:tasks.rados.rados.0.smithi026.stdout:564: seq_num 200 ranges {4464640=557056,5021696=401408} 2024-09-13T14:20:51.786 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:20:51.787 INFO:tasks.rados.rados.0.smithi026.stdout:564: writing smithi02683976-6 from 4464640 to 5021696 tid 1 2024-09-13T14:20:51.789 INFO:tasks.rados.rados.0.smithi026.stdout:564: writing smithi02683976-6 from 5021696 to 5423104 tid 2 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:565: copy_from oid 41 from oid 46 current snap is 48 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:562: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:562: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:562: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 80 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:562: done (3 left) 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:566: rollback oid 1 current snap is 48 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 44 2024-09-13T14:20:51.790 INFO:tasks.rados.rados.0.smithi026.stdout:561: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:20:51.863 INFO:tasks.rados.rados.0.smithi026.stdout:564: finishing write tid 1 to smithi02683976-6 2024-09-13T14:20:51.863 INFO:tasks.rados.rados.0.smithi026.stdout:566: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 0 (ObjNum 321 snap 0 seq_num 3890191888) dirty dne 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:564: finishing write tid 2 to smithi02683976-6 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:564: finishing write tid 3 to smithi02683976-6 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:564: finishing write tid 4 to smithi02683976-6 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 278 (ObjNum 200 snap 48 seq_num 200) dirty exists 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:564: left oid 6 (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:565: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:565: finishing copy_from to smithi02683976-41 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 117 (ObjNum 120 snap 27 seq_num 120) dirty exists 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:561: done (3 left) 2024-09-13T14:20:51.864 INFO:tasks.rados.rados.0.smithi026.stdout:564: done (2 left) 2024-09-13T14:20:51.865 INFO:tasks.rados.rados.0.smithi026.stdout:565: done (1 left) 2024-09-13T14:20:51.865 INFO:tasks.rados.rados.0.smithi026.stdout:566: done (0 left) 2024-09-13T14:20:51.865 INFO:tasks.rados.rados.0.smithi026.stdout:567: snap_remove snap 42 2024-09-13T14:20:51.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:20:51.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: Standby manager daemon y started 2024-09-13T14:20:51.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:20:51.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: Standby manager daemon y started 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:20:51.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:51 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:52.116 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:20:52.117 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: Standby manager daemon y started 2024-09-13T14:20:52.117 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:20:52.117 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:20:52.117 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:20:52.117 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:51 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/178805971' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 0s ago 4m 29.1M - 0.25.0 c8568f914cd2 52eac4b9b3e5 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (4m) 10s ago 4m 88.7M - 9.4.7 954c08fa6188 b034aa056559 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (5m) 0s ago 5m 13.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (5m) 10s ago 5m 18.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283 running (15s) 10s ago 9m 69.0M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4ae2b625819 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283,8765 running (10m) 0s ago 10m 274M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 90a812e595d2 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 0s ago 10m 56.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (9m) 10s ago 9m 40.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:20:52.451 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 0s ago 9m 46.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 0s ago 3m 16.2M - 1.5.0 0da6a335fe13 0720764634c2 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (3m) 10s ago 3m 15.4M - 1.5.0 0da6a335fe13 02ae27c8e5f5 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 0s ago 8m 112M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) 0s ago 8m 99.6M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (8m) 0s ago 8m 107M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 0s ago 7m 67.3M 2993M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (7m) 10s ago 7m 109M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (6m) 10s ago 6m 95.3M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (6m) 10s ago 6m 71.4M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (6m) 10s ago 6m 72.9M 2992M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:20:52.452 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (3m) 10s ago 4m 51.9M - 2.43.0 a07b618ecd1d 33771513b4fc 2024-09-13T14:20:52.764 INFO:tasks.rados.rados.0.smithi026.stdout:567: done (0 left) 2024-09-13T14:20:52.764 INFO:tasks.rados.rados.0.smithi026.stdout:568: snap_remove snap 46 2024-09-13T14:20:52.808 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:20:52.808 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:20:52.808 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:20:52.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 14, 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:20:52.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:20:52.810 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:52] "GET /metrics HTTP/1.1" 200 33178 "" "Prometheus/2.43.0" 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:20:53.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: pgmap v12: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.022 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.023 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.023 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:52 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.104 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/20 daemons upgraded", 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:20:53.105 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: pgmap v12: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: pgmap v12: 121 pgs: 1 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 115 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:53.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:52 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:53.774 INFO:tasks.rados.rados.0.smithi026.stdout:568: done (0 left) 2024-09-13T14:20:53.774 INFO:tasks.rados.rados.0.smithi026.stdout:569: snap_create 2024-09-13T14:20:54.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi026 to 2481M 2024-09-13T14:20:54.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:20:54.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/4293854604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi026 to 2481M 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/4293854604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:53 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi026 to 2481M 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/4293854604' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:53 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:54.776 INFO:tasks.rados.rados.0.smithi026.stdout:569: done (0 left) 2024-09-13T14:20:54.777 INFO:tasks.rados.rados.0.smithi026.stdout:570: read oid 47 snap -1 2024-09-13T14:20:54.777 INFO:tasks.rados.rados.0.smithi026.stdout:570: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:20:54.777 INFO:tasks.rados.rados.0.smithi026.stdout:571: delete oid 21 current snap is 49 2024-09-13T14:20:54.780 INFO:tasks.rados.rados.0.smithi026.stdout:571: done (1 left) 2024-09-13T14:20:54.780 INFO:tasks.rados.rados.0.smithi026.stdout:572: snap_create 2024-09-13T14:20:54.787 INFO:tasks.rados.rados.0.smithi026.stdout:570: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:20:55.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[39182]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:20:55.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[39182]: pgmap v15: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:55.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[27770]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:20:55.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:54 smithi026 ceph-mon[27770]: pgmap v15: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:55.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:20:55 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:20:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:20:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:54 smithi145 ceph-mon[31057]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:20:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:54 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:54 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:20:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:54 smithi145 ceph-mon[31057]: pgmap v15: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:570: done (1 left) 2024-09-13T14:20:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:572: done (0 left) 2024-09-13T14:20:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:573: setattr oid 7 current snap is 50 2024-09-13T14:20:55.787 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 50 2024-09-13T14:20:55.787 INFO:tasks.rados.rados.0.smithi026.stdout:574: seq_num 202 ranges {2326528=589824,2916352=401408} 2024-09-13T14:20:55.803 INFO:tasks.rados.rados.0.smithi026.stdout:574: writing smithi02683976-19 from 2326528 to 2916352 tid 1 2024-09-13T14:20:55.806 INFO:tasks.rados.rados.0.smithi026.stdout:574: writing smithi02683976-19 from 2916352 to 3317760 tid 2 2024-09-13T14:20:55.807 INFO:tasks.rados.rados.0.smithi026.stdout:575: copy_from oid 13 from oid 8 current snap is 50 2024-09-13T14:20:55.807 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 28 (ObjNum 129 snap 0 seq_num 519448984) dirty exists 2024-09-13T14:20:55.807 INFO:tasks.rados.rados.0.smithi026.stdout:573: done (2 left) 2024-09-13T14:20:55.807 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 50 2024-09-13T14:20:55.807 INFO:tasks.rados.rados.0.smithi026.stdout:576: seq_num 203 ranges {2326528=606208,2932736=401408} 2024-09-13T14:20:55.825 INFO:tasks.rados.rados.0.smithi026.stdout:576: writing smithi02683976-41 from 2326528 to 2932736 tid 1 2024-09-13T14:20:55.828 INFO:tasks.rados.rados.0.smithi026.stdout:576: writing smithi02683976-41 from 2932736 to 3334144 tid 2 2024-09-13T14:20:55.828 INFO:tasks.rados.rados.0.smithi026.stdout:577: read oid 20 snap 43 2024-09-13T14:20:55.828 INFO:tasks.rados.rados.0.smithi026.stdout:577: expect (ObjNum 518998096 snap 21981 seq_num 519334784) 2024-09-13T14:20:55.828 INFO:tasks.rados.rados.0.smithi026.stdout:578: snap_remove snap 45 2024-09-13T14:20:55.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:55 smithi145 ceph-mon[31057]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:20:56.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:55 smithi026 ceph-mon[27770]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:20:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:55 smithi026 ceph-mon[39182]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:20:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:574: finishing write tid 1 to smithi02683976-19 2024-09-13T14:20:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:574: finishing write tid 2 to smithi02683976-19 2024-09-13T14:20:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:574: finishing write tid 3 to smithi02683976-19 2024-09-13T14:20:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:574: finishing write tid 4 to smithi02683976-19 2024-09-13T14:20:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 290 (ObjNum 202 snap 50 seq_num 202) dirty exists 2024-09-13T14:20:56.795 INFO:tasks.rados.rados.0.smithi026.stdout:574: left oid 19 (ObjNum 202 snap 50 seq_num 202) 2024-09-13T14:20:56.795 INFO:tasks.rados.rados.0.smithi026.stdout:575: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:20:56.795 INFO:tasks.rados.rados.0.smithi026.stdout:575: finishing copy_from to smithi02683976-13 2024-09-13T14:20:56.795 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 201 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:20:56.795 INFO:tasks.rados.rados.0.smithi026.stdout:577: expect (ObjNum 518998096 snap 21981 seq_num 519334784) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: finishing write tid 1 to smithi02683976-41 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: finishing write tid 2 to smithi02683976-41 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: finishing write tid 3 to smithi02683976-41 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: finishing write tid 4 to smithi02683976-41 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 202 (ObjNum 203 snap 50 seq_num 203) dirty exists 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: left oid 41 (ObjNum 203 snap 50 seq_num 203) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:574: done (4 left) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:575: done (3 left) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:576: done (2 left) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:577: done (1 left) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:578: done (0 left) 2024-09-13T14:20:56.797 INFO:tasks.rados.rados.0.smithi026.stdout:579: read oid 46 snap -1 2024-09-13T14:20:56.798 INFO:tasks.rados.rados.0.smithi026.stdout:579: expect (ObjNum 120 snap 27 seq_num 120) 2024-09-13T14:20:56.798 INFO:tasks.rados.rados.0.smithi026.stdout:580: rollback oid 42 current snap is 50 2024-09-13T14:20:56.798 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 44 2024-09-13T14:20:56.798 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 50 2024-09-13T14:20:56.798 INFO:tasks.rados.rados.0.smithi026.stdout:581: seq_num 204 ranges {0=622592,622592=401408} 2024-09-13T14:20:56.801 INFO:tasks.rados.rados.0.smithi026.stdout:581: writing smithi02683976-26 from 0 to 622592 tid 1 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:581: writing smithi02683976-26 from 622592 to 1024000 tid 2 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:582: copy_from oid 41 from oid 40 current snap is 50 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:583: rollback oid 37 current snap is 50 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 48 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:580: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:580: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:584: copy_from oid 29 from oid 22 current snap is 50 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:580: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 229 (ObjNum 116 snap 24 seq_num 116) dirty exists 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:580: done (5 left) 2024-09-13T14:20:56.806 INFO:tasks.rados.rados.0.smithi026.stdout:585: rollback oid 45 current snap is 50 2024-09-13T14:20:56.807 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 47 2024-09-13T14:20:56.807 INFO:tasks.rados.rados.0.smithi026.stdout:586: delete oid 7 current snap is 50 2024-09-13T14:20:56.810 INFO:tasks.rados.rados.0.smithi026.stdout:583: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:20:56.811 INFO:tasks.rados.rados.0.smithi026.stdout:583: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:20:56.811 INFO:tasks.rados.rados.0.smithi026.stdout:583: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:20:56.811 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 232 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:20:56.812 INFO:tasks.rados.rados.0.smithi026.stdout:581: finishing write tid 1 to smithi02683976-26 2024-09-13T14:20:56.812 INFO:tasks.rados.rados.0.smithi026.stdout:583: done (6 left) 2024-09-13T14:20:56.813 INFO:tasks.rados.rados.0.smithi026.stdout:586: done (5 left) 2024-09-13T14:20:56.813 INFO:tasks.rados.rados.0.smithi026.stdout:587: delete oid 33 current snap is 50 2024-09-13T14:20:56.814 INFO:tasks.rados.rados.0.smithi026.stdout:585: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:20:56.815 INFO:tasks.rados.rados.0.smithi026.stdout:585: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:20:56.815 INFO:tasks.rados.rados.0.smithi026.stdout:579: expect (ObjNum 120 snap 27 seq_num 120) 2024-09-13T14:20:56.891 INFO:tasks.rados.rados.0.smithi026.stdout:581: finishing write tid 2 to smithi02683976-26 2024-09-13T14:20:56.891 INFO:tasks.rados.rados.0.smithi026.stdout:585: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:20:56.891 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 62 (ObjNum 107 snap 22 seq_num 107) dirty exists 2024-09-13T14:20:56.891 INFO:tasks.rados.rados.0.smithi026.stdout:581: finishing write tid 3 to smithi02683976-26 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:579: done (5 left) 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:585: done (4 left) 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:587: done (3 left) 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:588: read oid 14 snap -1 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:588: expect (ObjNum 189 snap 44 seq_num 189) 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:589: rollback oid 13 current snap is 50 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 49 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:581: finishing write tid 4 to smithi02683976-26 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 128 (ObjNum 204 snap 50 seq_num 204) dirty exists 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:581: left oid 26 (ObjNum 204 snap 50 seq_num 204) 2024-09-13T14:20:56.892 INFO:tasks.rados.rados.0.smithi026.stdout:584: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:584: finishing copy_from to smithi02683976-29 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 76 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:582: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:582: finishing copy_from to smithi02683976-41 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 155 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:581: done (4 left) 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:582: done (3 left) 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:584: done (2 left) 2024-09-13T14:20:56.893 INFO:tasks.rados.rados.0.smithi026.stdout:590: delete oid 50 current snap is 50 2024-09-13T14:20:56.894 INFO:tasks.rados.rados.0.smithi026.stdout:590: done (2 left) 2024-09-13T14:20:56.894 INFO:tasks.rados.rados.0.smithi026.stdout:591: read oid 10 snap -1 2024-09-13T14:20:56.894 INFO:tasks.rados.rados.0.smithi026.stdout:591: expect (ObjNum 133 snap 30 seq_num 133) 2024-09-13T14:20:56.894 INFO:tasks.rados.rados.0.smithi026.stdout:592: snap_create 2024-09-13T14:20:56.894 INFO:tasks.rados.rados.0.smithi026.stdout:589: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:20:56.895 INFO:tasks.rados.rados.0.smithi026.stdout:589: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:20:56.895 INFO:tasks.rados.rados.0.smithi026.stdout:589: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:20:56.895 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 303 (ObjNum 176 snap 41 seq_num 176) dirty exists 2024-09-13T14:20:56.906 INFO:tasks.rados.rados.0.smithi026.stdout:588: expect (ObjNum 189 snap 44 seq_num 189) 2024-09-13T14:20:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:56 smithi145 ceph-mon[31057]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:20:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:56 smithi145 ceph-mon[31057]: pgmap v18: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:57.151 INFO:tasks.rados.rados.0.smithi026.stdout:591: expect (ObjNum 133 snap 30 seq_num 133) 2024-09-13T14:20:57.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:56 smithi026 ceph-mon[39182]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:20:57.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:56 smithi026 ceph-mon[39182]: pgmap v18: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:57.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:56 smithi026 ceph-mon[27770]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:20:57.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:56 smithi026 ceph-mon[27770]: pgmap v18: 121 pgs: 121 active+clean; 99 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.9 MiB/s wr, 40 op/s 2024-09-13T14:20:57.804 INFO:tasks.rados.rados.0.smithi026.stdout:588: done (3 left) 2024-09-13T14:20:57.804 INFO:tasks.rados.rados.0.smithi026.stdout:589: done (2 left) 2024-09-13T14:20:57.804 INFO:tasks.rados.rados.0.smithi026.stdout:591: done (1 left) 2024-09-13T14:20:57.804 INFO:tasks.rados.rados.0.smithi026.stdout:592: done (0 left) 2024-09-13T14:20:57.804 INFO:tasks.rados.rados.0.smithi026.stdout:593: delete oid 27 current snap is 51 2024-09-13T14:20:57.808 INFO:tasks.rados.rados.0.smithi026.stdout:593: done (0 left) 2024-09-13T14:20:57.808 INFO:tasks.rados.rados.0.smithi026.stdout:594: snap_create 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:57 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:20:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:20:58.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:57 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:20:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:594: done (0 left) 2024-09-13T14:20:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:595: read oid 45 snap -1 2024-09-13T14:20:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:595: expect (ObjNum 107 snap 22 seq_num 107) 2024-09-13T14:20:58.743 INFO:tasks.rados.rados.0.smithi026.stdout:596: delete oid 23 current snap is 52 2024-09-13T14:20:58.749 INFO:tasks.rados.rados.0.smithi026.stdout:596: done (1 left) 2024-09-13T14:20:58.749 INFO:tasks.rados.rados.0.smithi026.stdout:597: setattr oid 31 current snap is 52 2024-09-13T14:20:58.751 INFO:tasks.rados.rados.0.smithi026.stdout:598: read oid 17 snap 48 2024-09-13T14:20:58.752 INFO:tasks.rados.rados.0.smithi026.stdout:598: expect (ObjNum 180 snap 44 seq_num 180) 2024-09-13T14:20:58.752 INFO:tasks.rados.rados.0.smithi026.stdout:599: copy_from oid 47 from oid 27 current snap is 52 2024-09-13T14:20:58.752 INFO:tasks.rados.rados.0.smithi026.stdout:600: snap_remove snap 44 2024-09-13T14:20:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:58 smithi145 ceph-mon[31057]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-13T14:20:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:58 smithi145 ceph-mon[31057]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:20:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:58 smithi145 ceph-mon[31057]: pgmap v21: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:20:58.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:20:58 smithi145 ceph-mon[31057]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:20:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[39182]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-13T14:20:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[39182]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[39182]: pgmap v21: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[39182]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[27770]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[27770]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[27770]: pgmap v21: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:20:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:20:58 smithi026 ceph-mon[27770]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:20:59.745 INFO:tasks.rados.rados.0.smithi026.stdout:600: done (4 left) 2024-09-13T14:20:59.745 INFO:tasks.rados.rados.0.smithi026.stdout:601: snap_remove snap 43 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:601: done (4 left) 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:602: rollback oid 50 current snap is 52 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 51 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:599: finishing copy_from to smithi02683976-47 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:599: got expected ENOENT (src dne) 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:599: finishing copy_from racing read to smithi02683976-47 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 304 (ObjNum 134 snap 30 seq_num 134) dirty exists 2024-09-13T14:21:00.753 INFO:tasks.rados.rados.0.smithi026.stdout:595: expect (ObjNum 107 snap 22 seq_num 107) 2024-09-13T14:21:00.844 INFO:tasks.rados.rados.0.smithi026.stdout:598: expect (ObjNum 180 snap 44 seq_num 180) 2024-09-13T14:21:01.110 INFO:tasks.rados.rados.0.smithi026.stdout:602: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:21:01.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 302 (ObjNum 129 snap 0 seq_num 1126316893) dirty dne 2024-09-13T14:21:01.110 INFO:tasks.rados.rados.0.smithi026.stdout:595: done (4 left) 2024-09-13T14:21:01.110 INFO:tasks.rados.rados.0.smithi026.stdout:597: done (3 left) 2024-09-13T14:21:01.110 INFO:tasks.rados.rados.0.smithi026.stdout:598: done (2 left) 2024-09-13T14:21:01.111 INFO:tasks.rados.rados.0.smithi026.stdout:599: done (1 left) 2024-09-13T14:21:01.111 INFO:tasks.rados.rados.0.smithi026.stdout:602: done (0 left) 2024-09-13T14:21:01.111 INFO:tasks.rados.rados.0.smithi026.stdout:603: delete oid 17 current snap is 52 2024-09-13T14:21:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:603: done (0 left) 2024-09-13T14:21:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:604: read oid 42 snap -1 2024-09-13T14:21:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:604: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:21:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:605: setattr oid 5 current snap is 52 2024-09-13T14:21:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:606: rmattr oid 26 current snap is 52 2024-09-13T14:21:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:606: done (2 left) 2024-09-13T14:21:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 52 2024-09-13T14:21:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:607: seq_num 211 ranges {0=737280,737280=401408} 2024-09-13T14:21:01.119 INFO:tasks.rados.rados.0.smithi026.stdout:607: writing smithi02683976-38 from 0 to 737280 tid 1 2024-09-13T14:21:01.123 INFO:tasks.rados.rados.0.smithi026.stdout:607: writing smithi02683976-38 from 737280 to 1138688 tid 2 2024-09-13T14:21:01.123 INFO:tasks.rados.rados.0.smithi026.stdout:608: snap_create 2024-09-13T14:21:01.123 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 361 (ObjNum 129 snap 0 seq_num 519592232) dirty exists 2024-09-13T14:21:01.123 INFO:tasks.rados.rados.0.smithi026.stdout:604: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:21:01.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:00 smithi026 ceph-mon[39182]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:21:01.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:00 smithi026 ceph-mon[39182]: pgmap v24: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:21:01.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:00 smithi026 ceph-mon[27770]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:21:01.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:00 smithi026 ceph-mon[27770]: pgmap v24: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:21:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:00 smithi145 ceph-mon[31057]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:21:01.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:00 smithi145 ceph-mon[31057]: pgmap v24: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-13T14:21:01.247 INFO:tasks.rados.rados.0.smithi026.stdout:607: finishing write tid 1 to smithi02683976-38 2024-09-13T14:21:01.247 INFO:tasks.rados.rados.0.smithi026.stdout:607: finishing write tid 2 to smithi02683976-38 2024-09-13T14:21:01.247 INFO:tasks.rados.rados.0.smithi026.stdout:607: finishing write tid 3 to smithi02683976-38 2024-09-13T14:21:01.247 INFO:tasks.rados.rados.0.smithi026.stdout:607: finishing write tid 4 to smithi02683976-38 2024-09-13T14:21:01.247 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 279 (ObjNum 211 snap 52 seq_num 211) dirty exists 2024-09-13T14:21:01.248 INFO:tasks.rados.rados.0.smithi026.stdout:607: left oid 38 (ObjNum 211 snap 52 seq_num 211) 2024-09-13T14:21:01.755 INFO:tasks.rados.rados.0.smithi026.stdout:604: done (3 left) 2024-09-13T14:21:01.755 INFO:tasks.rados.rados.0.smithi026.stdout:605: done (2 left) 2024-09-13T14:21:01.755 INFO:tasks.rados.rados.0.smithi026.stdout:607: done (1 left) 2024-09-13T14:21:01.755 INFO:tasks.rados.rados.0.smithi026.stdout:608: done (0 left) 2024-09-13T14:21:01.755 INFO:tasks.rados.rados.0.smithi026.stdout:609: snap_remove snap 47 2024-09-13T14:21:02.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:01 smithi026 ceph-mon[27770]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:21:02.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:01 smithi026 ceph-mon[39182]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:21:02.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:01 smithi145 ceph-mon[31057]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:21:02.757 INFO:tasks.rados.rados.0.smithi026.stdout:609: done (0 left) 2024-09-13T14:21:02.757 INFO:tasks.rados.rados.0.smithi026.stdout:610: snap_remove snap 50 2024-09-13T14:21:03.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[27770]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:21:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:03.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[27770]: pgmap v27: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[39182]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:21:03.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:03.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:02 smithi026 ceph-mon[39182]: pgmap v27: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:02] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-09-13T14:21:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:02 smithi145 ceph-mon[31057]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:21:03.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:02 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:03.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:02 smithi145 ceph-mon[31057]: pgmap v27: 121 pgs: 121 active+clean; 109 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.770 INFO:tasks.rados.rados.0.smithi026.stdout:610: done (0 left) 2024-09-13T14:21:03.770 INFO:tasks.rados.rados.0.smithi026.stdout:611: rollback oid 12 current snap is 53 2024-09-13T14:21:03.770 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 52 2024-09-13T14:21:03.770 INFO:tasks.rados.rados.0.smithi026.stdout:612: read oid 21 snap -1 2024-09-13T14:21:03.771 INFO:tasks.rados.rados.0.smithi026.stdout:612: expect deleted 2024-09-13T14:21:03.771 INFO:tasks.rados.rados.0.smithi026.stdout:613: snap_remove snap 51 2024-09-13T14:21:04.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:03 smithi026 ceph-mon[39182]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:21:04.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:03 smithi026 ceph-mon[27770]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:21:04.175 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:03 smithi145 ceph-mon[31057]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:21:04.777 INFO:tasks.rados.rados.0.smithi026.stdout:613: done (2 left) 2024-09-13T14:21:04.777 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 53 2024-09-13T14:21:04.778 INFO:tasks.rados.rados.0.smithi026.stdout:614: seq_num 212 ranges {573440=753664,1327104=401408} 2024-09-13T14:21:04.784 INFO:tasks.rados.rados.0.smithi026.stdout:614: writing smithi02683976-44 from 573440 to 1327104 tid 1 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:614: writing smithi02683976-44 from 1327104 to 1728512 tid 2 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 53 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:611: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:611: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:611: finishing rollback tid 2 to smithi02683976-12 2024-09-13T14:21:04.787 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 132 (ObjNum 157 snap 37 seq_num 157) dirty exists 2024-09-13T14:21:04.788 INFO:tasks.rados.rados.0.smithi026.stdout:615: seq_num 213 ranges {3653632=770048,4423680=401408} 2024-09-13T14:21:04.811 INFO:tasks.rados.rados.0.smithi026.stdout:615: writing smithi02683976-28 from 3653632 to 4423680 tid 1 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:615: writing smithi02683976-28 from 4423680 to 4825088 tid 2 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:611: done (3 left) 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:612: done (2 left) 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 53 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:614: finishing write tid 1 to smithi02683976-44 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:614: finishing write tid 2 to smithi02683976-44 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:614: finishing write tid 3 to smithi02683976-44 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:614: finishing write tid 4 to smithi02683976-44 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 160 (ObjNum 212 snap 53 seq_num 212) dirty exists 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:614: left oid 44 (ObjNum 212 snap 53 seq_num 212) 2024-09-13T14:21:04.814 INFO:tasks.rados.rados.0.smithi026.stdout:616: seq_num 214 ranges {1138688=794624,1933312=401408} 2024-09-13T14:21:04.825 INFO:tasks.rados.rados.0.smithi026.stdout:616: writing smithi02683976-38 from 1138688 to 1933312 tid 1 2024-09-13T14:21:04.827 INFO:tasks.rados.rados.0.smithi026.stdout:616: writing smithi02683976-38 from 1933312 to 2334720 tid 2 2024-09-13T14:21:04.828 INFO:tasks.rados.rados.0.smithi026.stdout:614: done (2 left) 2024-09-13T14:21:04.828 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 53 2024-09-13T14:21:04.828 INFO:tasks.rados.rados.0.smithi026.stdout:615: finishing write tid 1 to smithi02683976-28 2024-09-13T14:21:04.828 INFO:tasks.rados.rados.0.smithi026.stdout:617: seq_num 215 ranges {3317760=409600,3727360=532480,4259840=270336} 2024-09-13T14:21:04.848 INFO:tasks.rados.rados.0.smithi026.stdout:617: writing smithi02683976-19 from 3317760 to 3727360 tid 1 2024-09-13T14:21:04.851 INFO:tasks.rados.rados.0.smithi026.stdout:617: writing smithi02683976-19 from 3727360 to 4259840 tid 2 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:617: writing smithi02683976-19 from 4259840 to 4530176 tid 3 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 53 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:615: finishing write tid 2 to smithi02683976-28 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:615: finishing write tid 3 to smithi02683976-28 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:615: finishing write tid 4 to smithi02683976-28 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 214 (ObjNum 213 snap 53 seq_num 213) dirty exists 2024-09-13T14:21:04.853 INFO:tasks.rados.rados.0.smithi026.stdout:615: left oid 28 (ObjNum 213 snap 53 seq_num 213) 2024-09-13T14:21:04.854 INFO:tasks.rados.rados.0.smithi026.stdout:618: seq_num 216 ranges {3653632=425984,4079616=409600,4489216=393216} 2024-09-13T14:21:04.875 INFO:tasks.rados.rados.0.smithi026.stdout:618: writing smithi02683976-14 from 3653632 to 4079616 tid 1 2024-09-13T14:21:04.877 INFO:tasks.rados.rados.0.smithi026.stdout:618: writing smithi02683976-14 from 4079616 to 4489216 tid 2 2024-09-13T14:21:04.880 INFO:tasks.rados.rados.0.smithi026.stdout:618: writing smithi02683976-14 from 4489216 to 4882432 tid 3 2024-09-13T14:21:04.880 INFO:tasks.rados.rados.0.smithi026.stdout:615: done (3 left) 2024-09-13T14:21:04.880 INFO:tasks.rados.rados.0.smithi026.stdout:619: setattr oid 3 current snap is 53 2024-09-13T14:21:04.880 INFO:tasks.rados.rados.0.smithi026.stdout:616: finishing write tid 1 to smithi02683976-38 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:616: finishing write tid 2 to smithi02683976-38 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:616: finishing write tid 3 to smithi02683976-38 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:616: finishing write tid 4 to smithi02683976-38 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 282 (ObjNum 214 snap 53 seq_num 214) dirty exists 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:616: left oid 38 (ObjNum 214 snap 53 seq_num 214) 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: finishing write tid 1 to smithi02683976-19 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: finishing write tid 2 to smithi02683976-19 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: finishing write tid 3 to smithi02683976-19 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: finishing write tid 4 to smithi02683976-19 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: finishing write tid 5 to smithi02683976-19 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 332 (ObjNum 215 snap 53 seq_num 215) dirty exists 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:617: left oid 19 (ObjNum 215 snap 53 seq_num 215) 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:618: finishing write tid 1 to smithi02683976-14 2024-09-13T14:21:04.881 INFO:tasks.rados.rados.0.smithi026.stdout:618: finishing write tid 2 to smithi02683976-14 2024-09-13T14:21:04.882 INFO:tasks.rados.rados.0.smithi026.stdout:616: done (3 left) 2024-09-13T14:21:04.882 INFO:tasks.rados.rados.0.smithi026.stdout:617: done (2 left) 2024-09-13T14:21:04.882 INFO:tasks.rados.rados.0.smithi026.stdout:620: delete oid 7 current snap is 53 2024-09-13T14:21:04.882 INFO:tasks.rados.rados.0.smithi026.stdout:618: finishing write tid 3 to smithi02683976-14 2024-09-13T14:21:04.883 INFO:tasks.rados.rados.0.smithi026.stdout:620: done (2 left) 2024-09-13T14:21:04.883 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 53 2024-09-13T14:21:04.883 INFO:tasks.rados.rados.0.smithi026.stdout:621: seq_num 218 ranges {425984=458752,884736=581632,1466368=221184} 2024-09-13T14:21:04.887 INFO:tasks.rados.rados.0.smithi026.stdout:621: writing smithi02683976-25 from 425984 to 884736 tid 1 2024-09-13T14:21:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:621: writing smithi02683976-25 from 884736 to 1466368 tid 2 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:621: writing smithi02683976-25 from 1466368 to 1687552 tid 3 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:622: copy_from oid 45 from oid 6 current snap is 53 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:618: finishing write tid 4 to smithi02683976-14 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 375 (ObjNum 198 snap 47 seq_num 198) dirty exists 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:618: finishing write tid 5 to smithi02683976-14 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 287 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:618: left oid 14 (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:621: finishing write tid 1 to smithi02683976-25 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:618: done (3 left) 2024-09-13T14:21:04.893 INFO:tasks.rados.rados.0.smithi026.stdout:619: done (2 left) 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:623: rollback oid 9 current snap is 53 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 49 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:624: copy_from oid 32 from oid 18 current snap is 53 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:625: copy_from oid 17 from oid 7 current snap is 53 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 53 2024-09-13T14:21:04.894 INFO:tasks.rados.rados.0.smithi026.stdout:626: seq_num 219 ranges {0=475136,475136=466944,942080=335872} 2024-09-13T14:21:04.896 INFO:tasks.rados.rados.0.smithi026.stdout:626: writing smithi02683976-16 from 0 to 475136 tid 1 2024-09-13T14:21:04.899 INFO:tasks.rados.rados.0.smithi026.stdout:626: writing smithi02683976-16 from 475136 to 942080 tid 2 2024-09-13T14:21:04.901 INFO:tasks.rados.rados.0.smithi026.stdout:626: writing smithi02683976-16 from 942080 to 1277952 tid 3 2024-09-13T14:21:04.901 INFO:tasks.rados.rados.0.smithi026.stdout:627: snap_remove snap 52 2024-09-13T14:21:05.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:04 smithi026 ceph-mon[27770]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:21:05.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:04 smithi026 ceph-mon[27770]: pgmap v30: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:05.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:04 smithi026 ceph-mon[39182]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:21:05.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:04 smithi026 ceph-mon[39182]: pgmap v30: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:05.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:04 smithi145 ceph-mon[31057]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:21:05.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:04 smithi145 ceph-mon[31057]: pgmap v30: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:05.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:05.780 INFO:tasks.rados.rados.0.smithi026.stdout:627: done (6 left) 2024-09-13T14:21:05.780 INFO:tasks.rados.rados.0.smithi026.stdout:628: read oid 50 snap -1 2024-09-13T14:21:05.780 INFO:tasks.rados.rados.0.smithi026.stdout:628: expect deleted 2024-09-13T14:21:05.780 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 53 2024-09-13T14:21:05.780 INFO:tasks.rados.rados.0.smithi026.stdout:625: finishing copy_from to smithi02683976-17 2024-09-13T14:21:05.781 INFO:tasks.rados.rados.0.smithi026.stdout:625: got expected ENOENT (src dne) 2024-09-13T14:21:05.781 INFO:tasks.rados.rados.0.smithi026.stdout:629: seq_num 220 ranges {4530176=491520,5021696=753664,5775360=49152} 2024-09-13T14:21:05.808 INFO:tasks.rados.rados.0.smithi026.stdout:629: writing smithi02683976-19 from 4530176 to 5021696 tid 1 2024-09-13T14:21:05.813 INFO:tasks.rados.rados.0.smithi026.stdout:629: writing smithi02683976-19 from 5021696 to 5775360 tid 2 2024-09-13T14:21:05.813 INFO:tasks.rados.rados.0.smithi026.stdout:629: writing smithi02683976-19 from 5775360 to 5824512 tid 3 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:630: snap_create 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:621: finishing write tid 2 to smithi02683976-25 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:625: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:621: finishing write tid 3 to smithi02683976-25 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:621: finishing write tid 4 to smithi02683976-25 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:623: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:623: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:623: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 341 (ObjNum 161 snap 38 seq_num 161) dirty exists 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:621: finishing write tid 5 to smithi02683976-25 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 177 (ObjNum 218 snap 53 seq_num 218) dirty exists 2024-09-13T14:21:05.814 INFO:tasks.rados.rados.0.smithi026.stdout:621: left oid 25 (ObjNum 218 snap 53 seq_num 218) 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: finishing write tid 1 to smithi02683976-16 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: finishing write tid 2 to smithi02683976-16 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: finishing write tid 3 to smithi02683976-16 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: finishing write tid 4 to smithi02683976-16 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:624: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: finishing write tid 5 to smithi02683976-16 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 336 (ObjNum 219 snap 53 seq_num 219) dirty exists 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:626: left oid 16 (ObjNum 219 snap 53 seq_num 219) 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:624: finishing copy_from to smithi02683976-32 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 135 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:622: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:622: finishing copy_from to smithi02683976-45 2024-09-13T14:21:05.815 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 278 (ObjNum 200 snap 48 seq_num 200) dirty exists 2024-09-13T14:21:05.816 INFO:tasks.rados.rados.0.smithi026.stdout:629: finishing write tid 1 to smithi02683976-19 2024-09-13T14:21:05.817 INFO:tasks.rados.rados.0.smithi026.stdout:629: finishing write tid 2 to smithi02683976-19 2024-09-13T14:21:05.817 INFO:tasks.rados.rados.0.smithi026.stdout:629: finishing write tid 3 to smithi02683976-19 2024-09-13T14:21:05.817 INFO:tasks.rados.rados.0.smithi026.stdout:629: finishing write tid 4 to smithi02683976-19 2024-09-13T14:21:05.818 INFO:tasks.rados.rados.0.smithi026.stdout:629: finishing write tid 5 to smithi02683976-19 2024-09-13T14:21:05.818 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 342 (ObjNum 220 snap 53 seq_num 220) dirty exists 2024-09-13T14:21:05.818 INFO:tasks.rados.rados.0.smithi026.stdout:629: left oid 19 (ObjNum 220 snap 53 seq_num 220) 2024-09-13T14:21:06.016 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:05 smithi145 ceph-mon[31057]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:21:06.016 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:05 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.016 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:05 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[39182]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:21:06.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[27770]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:21:06.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:05 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:621: done (8 left) 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:622: done (7 left) 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:623: done (6 left) 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:624: done (5 left) 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:625: done (4 left) 2024-09-13T14:21:06.795 INFO:tasks.rados.rados.0.smithi026.stdout:626: done (3 left) 2024-09-13T14:21:06.796 INFO:tasks.rados.rados.0.smithi026.stdout:628: done (2 left) 2024-09-13T14:21:06.796 INFO:tasks.rados.rados.0.smithi026.stdout:629: done (1 left) 2024-09-13T14:21:06.796 INFO:tasks.rados.rados.0.smithi026.stdout:630: done (0 left) 2024-09-13T14:21:06.796 INFO:tasks.rados.rados.0.smithi026.stdout:631: delete oid 2 current snap is 54 2024-09-13T14:21:06.798 INFO:tasks.rados.rados.0.smithi026.stdout:631: done (0 left) 2024-09-13T14:21:06.798 INFO:tasks.rados.rados.0.smithi026.stdout:632: delete oid 5 current snap is 54 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:632: done (0 left) 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:633: read oid 43 snap -1 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:633: expect (ObjNum 186 snap 44 seq_num 186) 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:634: read oid 26 snap -1 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:634: expect (ObjNum 204 snap 50 seq_num 204) 2024-09-13T14:21:06.804 INFO:tasks.rados.rados.0.smithi026.stdout:635: setattr oid 7 current snap is 54 2024-09-13T14:21:06.806 INFO:tasks.rados.rados.0.smithi026.stdout:636: delete oid 42 current snap is 54 2024-09-13T14:21:06.808 INFO:tasks.rados.rados.0.smithi026.stdout:636: done (3 left) 2024-09-13T14:21:06.808 INFO:tasks.rados.rados.0.smithi026.stdout:637: snap_create 2024-09-13T14:21:06.809 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 31 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:21:06.813 INFO:tasks.rados.rados.0.smithi026.stdout:634: expect (ObjNum 204 snap 50 seq_num 204) 2024-09-13T14:21:06.842 INFO:tasks.rados.rados.0.smithi026.stdout:633: expect (ObjNum 186 snap 44 seq_num 186) 2024-09-13T14:21:07.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Adjusting osd_memory_target on smithi145 to 2480M 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: pgmap v33: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Adjusting osd_memory_target on smithi145 to 2480M 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: pgmap v33: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:06 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Adjusting osd_memory_target on smithi145 to 2480M 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:07.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: pgmap v33: 121 pgs: 121 active+clean; 95 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 278 KiB/s wr, 15 op/s 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:06 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:633: done (3 left) 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:634: done (2 left) 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:635: done (1 left) 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:637: done (0 left) 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:638: rollback oid 47 current snap is 55 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 53 2024-09-13T14:21:07.802 INFO:tasks.rados.rados.0.smithi026.stdout:639: read oid 10 snap -1 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:639: expect (ObjNum 133 snap 30 seq_num 133) 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:640: read oid 2 snap -1 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:640: expect deleted 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:641: rollback oid 16 current snap is 55 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 48 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 55 2024-09-13T14:21:07.803 INFO:tasks.rados.rados.0.smithi026.stdout:642: seq_num 224 ranges {942080=557056,1499136=737280,2236416=65536} 2024-09-13T14:21:07.810 INFO:tasks.rados.rados.0.smithi026.stdout:642: writing smithi02683976-3 from 942080 to 1499136 tid 1 2024-09-13T14:21:07.815 INFO:tasks.rados.rados.0.smithi026.stdout:642: writing smithi02683976-3 from 1499136 to 2236416 tid 2 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:642: writing smithi02683976-3 from 2236416 to 2301952 tid 3 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:643: read oid 35 snap -1 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:643: expect deleted 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:644: read oid 6 snap -1 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:644: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 55 2024-09-13T14:21:07.816 INFO:tasks.rados.rados.0.smithi026.stdout:645: seq_num 225 ranges {2416640=573440,2990080=622592,3612672=180224} 2024-09-13T14:21:07.833 INFO:tasks.rados.rados.0.smithi026.stdout:645: writing smithi02683976-36 from 2416640 to 2990080 tid 1 2024-09-13T14:21:07.837 INFO:tasks.rados.rados.0.smithi026.stdout:645: writing smithi02683976-36 from 2990080 to 3612672 tid 2 2024-09-13T14:21:07.838 INFO:tasks.rados.rados.0.smithi026.stdout:645: writing smithi02683976-36 from 3612672 to 3792896 tid 3 2024-09-13T14:21:07.838 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 55 2024-09-13T14:21:07.838 INFO:tasks.rados.rados.0.smithi026.stdout:646: seq_num 226 ranges {0=589824,589824=507904,1097728=294912} 2024-09-13T14:21:07.842 INFO:tasks.rados.rados.0.smithi026.stdout:646: writing smithi02683976-21 from 0 to 589824 tid 1 2024-09-13T14:21:07.844 INFO:tasks.rados.rados.0.smithi026.stdout:646: writing smithi02683976-21 from 589824 to 1097728 tid 2 2024-09-13T14:21:07.846 INFO:tasks.rados.rados.0.smithi026.stdout:646: writing smithi02683976-21 from 1097728 to 1392640 tid 3 2024-09-13T14:21:07.846 INFO:tasks.rados.rados.0.smithi026.stdout:647: snap_remove snap 49 2024-09-13T14:21:08.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:21:08.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[27770]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:21:08.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:07 smithi026 ceph-mon[39182]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:21:08.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:07 smithi145 ceph-mon[31057]: Deploying daemon alertmanager.a on smithi026 2024-09-13T14:21:08.745 INFO:tasks.rados.rados.0.smithi026.stdout:647: done (9 left) 2024-09-13T14:21:08.746 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 55 2024-09-13T14:21:08.746 INFO:tasks.rados.rados.0.smithi026.stdout:648: seq_num 227 ranges {4833280=606208,5439488=794624,6234112=8192} 2024-09-13T14:21:08.776 INFO:tasks.rados.rados.0.smithi026.stdout:648: writing smithi02683976-31 from 4833280 to 5439488 tid 1 2024-09-13T14:21:08.781 INFO:tasks.rados.rados.0.smithi026.stdout:648: writing smithi02683976-31 from 5439488 to 6234112 tid 2 2024-09-13T14:21:08.781 INFO:tasks.rados.rados.0.smithi026.stdout:648: writing smithi02683976-31 from 6234112 to 6242304 tid 3 2024-09-13T14:21:08.781 INFO:tasks.rados.rados.0.smithi026.stdout:640: done (9 left) 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 55 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:638: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:641: finishing rollback tid 0 to smithi02683976-16 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:641: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 345 (ObjNum 129 snap 0 seq_num 519592984) dirty dne 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:638: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:638: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 98 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-13T14:21:08.782 INFO:tasks.rados.rados.0.smithi026.stdout:639: expect (ObjNum 133 snap 30 seq_num 133) 2024-09-13T14:21:08.910 INFO:tasks.rados.rados.0.smithi026.stdout:642: finishing write tid 1 to smithi02683976-3 2024-09-13T14:21:08.911 INFO:tasks.rados.rados.0.smithi026.stdout:649: seq_num 228 ranges {0=622592,622592=671744,1294336=131072} 2024-09-13T14:21:08.914 INFO:tasks.rados.rados.0.smithi026.stdout:649: writing smithi02683976-42 from 0 to 622592 tid 1 2024-09-13T14:21:08.919 INFO:tasks.rados.rados.0.smithi026.stdout:649: writing smithi02683976-42 from 622592 to 1294336 tid 2 2024-09-13T14:21:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:649: writing smithi02683976-42 from 1294336 to 1425408 tid 3 2024-09-13T14:21:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:638: done (9 left) 2024-09-13T14:21:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:639: done (8 left) 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:641: done (7 left) 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:650: snap_create 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:642: finishing write tid 2 to smithi02683976-3 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:642: finishing write tid 3 to smithi02683976-3 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:642: finishing write tid 4 to smithi02683976-3 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:642: finishing write tid 5 to smithi02683976-3 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 384 (ObjNum 224 snap 55 seq_num 224) dirty exists 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:642: left oid 3 (ObjNum 224 snap 55 seq_num 224) 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:645: finishing write tid 1 to smithi02683976-36 2024-09-13T14:21:08.921 INFO:tasks.rados.rados.0.smithi026.stdout:644: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:09.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[39182]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:21:09.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[39182]: pgmap v36: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:09.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[39182]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:21:09.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[27770]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:21:09.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[27770]: pgmap v36: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:09.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:08 smithi026 ceph-mon[27770]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:21:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:08 smithi145 ceph-mon[31057]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:21:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:08 smithi145 ceph-mon[31057]: pgmap v36: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:08 smithi145 ceph-mon[31057]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:645: finishing write tid 2 to smithi02683976-36 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:645: finishing write tid 3 to smithi02683976-36 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:645: finishing write tid 4 to smithi02683976-36 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:645: finishing write tid 5 to smithi02683976-36 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 389 (ObjNum 225 snap 55 seq_num 225) dirty exists 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:645: left oid 36 (ObjNum 225 snap 55 seq_num 225) 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:646: finishing write tid 1 to smithi02683976-21 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:646: finishing write tid 2 to smithi02683976-21 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:646: finishing write tid 3 to smithi02683976-21 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:646: finishing write tid 4 to smithi02683976-21 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:646: finishing write tid 5 to smithi02683976-21 2024-09-13T14:21:09.275 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 347 (ObjNum 226 snap 55 seq_num 226) dirty exists 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:646: left oid 21 (ObjNum 226 snap 55 seq_num 226) 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: finishing write tid 1 to smithi02683976-31 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: finishing write tid 2 to smithi02683976-31 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: finishing write tid 3 to smithi02683976-31 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: finishing write tid 4 to smithi02683976-31 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: finishing write tid 5 to smithi02683976-31 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 358 (ObjNum 227 snap 55 seq_num 227) dirty exists 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:648: left oid 31 (ObjNum 227 snap 55 seq_num 227) 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:649: finishing write tid 1 to smithi02683976-42 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:649: finishing write tid 2 to smithi02683976-42 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:649: finishing write tid 3 to smithi02683976-42 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:649: finishing write tid 4 to smithi02683976-42 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:649: finishing write tid 5 to smithi02683976-42 2024-09-13T14:21:09.276 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 237 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:21:09.277 INFO:tasks.rados.rados.0.smithi026.stdout:649: left oid 42 (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:21:09.808 INFO:tasks.rados.rados.0.smithi026.stdout:642: done (7 left) 2024-09-13T14:21:09.808 INFO:tasks.rados.rados.0.smithi026.stdout:643: done (6 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:644: done (5 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:645: done (4 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:646: done (3 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:648: done (2 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:649: done (1 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:650: done (0 left) 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:651: rollback oid 13 current snap is 56 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 53 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 56 2024-09-13T14:21:09.809 INFO:tasks.rados.rados.0.smithi026.stdout:652: seq_num 229 ranges {0=638976,638976=606208,1245184=196608} 2024-09-13T14:21:09.813 INFO:tasks.rados.rados.0.smithi026.stdout:652: writing smithi02683976-35 from 0 to 638976 tid 1 2024-09-13T14:21:09.819 INFO:tasks.rados.rados.0.smithi026.stdout:652: writing smithi02683976-35 from 638976 to 1245184 tid 2 2024-09-13T14:21:09.820 INFO:tasks.rados.rados.0.smithi026.stdout:652: writing smithi02683976-35 from 1245184 to 1441792 tid 3 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:651: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:651: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:651: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 355 (ObjNum 176 snap 41 seq_num 176) dirty exists 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:651: done (1 left) 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:653: read oid 19 snap -1 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:653: expect (ObjNum 220 snap 53 seq_num 220) 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:654: copy_from oid 32 from oid 50 current snap is 56 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 56 2024-09-13T14:21:09.821 INFO:tasks.rados.rados.0.smithi026.stdout:655: seq_num 230 ranges {5423104=655360,6078464=491520,6569984=311296} 2024-09-13T14:21:09.854 INFO:tasks.rados.rados.0.smithi026.stdout:655: writing smithi02683976-6 from 5423104 to 6078464 tid 1 2024-09-13T14:21:09.857 INFO:tasks.rados.rados.0.smithi026.stdout:655: writing smithi02683976-6 from 6078464 to 6569984 tid 2 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:655: writing smithi02683976-6 from 6569984 to 6881280 tid 3 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:656: setattr oid 26 current snap is 56 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:652: finishing write tid 1 to smithi02683976-35 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:652: finishing write tid 2 to smithi02683976-35 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:652: finishing write tid 3 to smithi02683976-35 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:654: finishing copy_from to smithi02683976-32 2024-09-13T14:21:09.859 INFO:tasks.rados.rados.0.smithi026.stdout:654: got expected ENOENT (src dne) 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:654: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:652: finishing write tid 4 to smithi02683976-35 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:652: finishing write tid 5 to smithi02683976-35 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 403 (ObjNum 229 snap 56 seq_num 229) dirty exists 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:652: left oid 35 (ObjNum 229 snap 56 seq_num 229) 2024-09-13T14:21:09.860 INFO:tasks.rados.rados.0.smithi026.stdout:653: expect (ObjNum 220 snap 53 seq_num 220) 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: finishing write tid 1 to smithi02683976-6 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: finishing write tid 2 to smithi02683976-6 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: finishing write tid 3 to smithi02683976-6 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: finishing write tid 4 to smithi02683976-6 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: finishing write tid 5 to smithi02683976-6 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 363 (ObjNum 230 snap 56 seq_num 230) dirty exists 2024-09-13T14:21:10.196 INFO:tasks.rados.rados.0.smithi026.stdout:655: left oid 6 (ObjNum 230 snap 56 seq_num 230) 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:652: done (4 left) 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:653: done (3 left) 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:654: done (2 left) 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:655: done (1 left) 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:657: setattr oid 34 current snap is 56 2024-09-13T14:21:10.197 INFO:tasks.rados.rados.0.smithi026.stdout:658: snap_create 2024-09-13T14:21:10.199 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 140 (ObjNum 204 snap 50 seq_num 204) dirty exists 2024-09-13T14:21:10.199 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 226 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-13T14:21:10.812 INFO:tasks.rados.rados.0.smithi026.stdout:656: done (2 left) 2024-09-13T14:21:10.813 INFO:tasks.rados.rados.0.smithi026.stdout:657: done (1 left) 2024-09-13T14:21:10.813 INFO:tasks.rados.rados.0.smithi026.stdout:658: done (0 left) 2024-09-13T14:21:10.813 INFO:tasks.rados.rados.0.smithi026.stdout:659: rollback oid 32 current snap is 57 2024-09-13T14:21:10.813 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 55 2024-09-13T14:21:10.813 INFO:tasks.rados.rados.0.smithi026.stdout:660: snap_remove snap 56 2024-09-13T14:21:11.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:10 smithi026 ceph-mon[27770]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:21:11.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:10 smithi026 ceph-mon[27770]: pgmap v39: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:11.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:10 smithi026 ceph-mon[39182]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:21:11.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:10 smithi026 ceph-mon[39182]: pgmap v39: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:10 smithi145 ceph-mon[31057]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:21:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:10 smithi145 ceph-mon[31057]: pgmap v39: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 4.4 MiB/s wr, 24 op/s 2024-09-13T14:21:11.817 INFO:tasks.rados.rados.0.smithi026.stdout:660: done (1 left) 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 57 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:659: finishing rollback tid 0 to smithi02683976-32 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:659: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:659: finishing rollback tid 2 to smithi02683976-32 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 300 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:21:11.818 INFO:tasks.rados.rados.0.smithi026.stdout:661: seq_num 231 ranges {0=671744,671744=778240,1449984=24576} 2024-09-13T14:21:11.821 INFO:tasks.rados.rados.0.smithi026.stdout:661: writing smithi02683976-24 from 0 to 671744 tid 1 2024-09-13T14:21:11.826 INFO:tasks.rados.rados.0.smithi026.stdout:661: writing smithi02683976-24 from 671744 to 1449984 tid 2 2024-09-13T14:21:11.827 INFO:tasks.rados.rados.0.smithi026.stdout:661: writing smithi02683976-24 from 1449984 to 1474560 tid 3 2024-09-13T14:21:11.827 INFO:tasks.rados.rados.0.smithi026.stdout:659: done (1 left) 2024-09-13T14:21:11.827 INFO:tasks.rados.rados.0.smithi026.stdout:662: copy_from oid 35 from oid 27 current snap is 57 2024-09-13T14:21:11.827 INFO:tasks.rados.rados.0.smithi026.stdout:663: snap_create 2024-09-13T14:21:11.829 INFO:tasks.rados.rados.0.smithi026.stdout:661: finishing write tid 1 to smithi02683976-24 2024-09-13T14:21:11.831 INFO:tasks.rados.rados.0.smithi026.stdout:662: finishing copy_from to smithi02683976-35 2024-09-13T14:21:11.831 INFO:tasks.rados.rados.0.smithi026.stdout:662: got expected ENOENT (src dne) 2024-09-13T14:21:11.831 INFO:tasks.rados.rados.0.smithi026.stdout:662: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:21:11.832 INFO:tasks.rados.rados.0.smithi026.stdout:661: finishing write tid 2 to smithi02683976-24 2024-09-13T14:21:11.832 INFO:tasks.rados.rados.0.smithi026.stdout:661: finishing write tid 3 to smithi02683976-24 2024-09-13T14:21:11.832 INFO:tasks.rados.rados.0.smithi026.stdout:661: finishing write tid 4 to smithi02683976-24 2024-09-13T14:21:11.833 INFO:tasks.rados.rados.0.smithi026.stdout:661: finishing write tid 5 to smithi02683976-24 2024-09-13T14:21:11.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 189 (ObjNum 231 snap 57 seq_num 231) dirty exists 2024-09-13T14:21:11.833 INFO:tasks.rados.rados.0.smithi026.stdout:661: left oid 24 (ObjNum 231 snap 57 seq_num 231) 2024-09-13T14:21:12.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:11 smithi026 ceph-mon[39182]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:21:12.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:11 smithi026 ceph-mon[27770]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:21:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:11 smithi145 ceph-mon[31057]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:21:12.824 INFO:tasks.rados.rados.0.smithi026.stdout:661: done (2 left) 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:662: done (1 left) 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:663: done (0 left) 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:664: read oid 39 snap -1 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:664: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:665: read oid 8 snap 53 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:665: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:666: snap_remove snap 55 2024-09-13T14:21:13.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:12 smithi026 ceph-mon[39182]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:21:13.083 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:12 smithi026 ceph-mon[39182]: pgmap v42: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:12 smithi026 ceph-mon[27770]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:21:13.083 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:12 smithi026 ceph-mon[27770]: pgmap v42: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:12 smithi145 ceph-mon[31057]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:21:13.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:12 smithi145 ceph-mon[31057]: pgmap v42: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:12] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-09-13T14:21:13.827 INFO:tasks.rados.rados.0.smithi026.stdout:666: done (2 left) 2024-09-13T14:21:13.827 INFO:tasks.rados.rados.0.smithi026.stdout:667: delete oid 35 current snap is 58 2024-09-13T14:21:13.827 INFO:tasks.rados.rados.0.smithi026.stdout:665: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:13.852 INFO:tasks.rados.rados.0.smithi026.stdout:664: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:13.934 INFO:tasks.rados.rados.0.smithi026.stdout:664: done (2 left) 2024-09-13T14:21:13.934 INFO:tasks.rados.rados.0.smithi026.stdout:665: done (1 left) 2024-09-13T14:21:13.934 INFO:tasks.rados.rados.0.smithi026.stdout:667: done (0 left) 2024-09-13T14:21:13.935 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 58 2024-09-13T14:21:13.935 INFO:tasks.rados.rados.0.smithi026.stdout:668: seq_num 233 ranges {1024000=712704,1736704=548864,2285568=253952} 2024-09-13T14:21:13.943 INFO:tasks.rados.rados.0.smithi026.stdout:668: writing smithi02683976-26 from 1024000 to 1736704 tid 1 2024-09-13T14:21:13.946 INFO:tasks.rados.rados.0.smithi026.stdout:668: writing smithi02683976-26 from 1736704 to 2285568 tid 2 2024-09-13T14:21:13.948 INFO:tasks.rados.rados.0.smithi026.stdout:668: writing smithi02683976-26 from 2285568 to 2539520 tid 3 2024-09-13T14:21:13.948 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 58 2024-09-13T14:21:13.948 INFO:tasks.rados.rados.0.smithi026.stdout:668: finishing write tid 1 to smithi02683976-26 2024-09-13T14:21:13.948 INFO:tasks.rados.rados.0.smithi026.stdout:669: seq_num 234 ranges {6881280=729088,7610368=434176,8044544=368640} 2024-09-13T14:21:13.989 INFO:tasks.rados.rados.0.smithi026.stdout:669: writing smithi02683976-6 from 6881280 to 7610368 tid 1 2024-09-13T14:21:13.992 INFO:tasks.rados.rados.0.smithi026.stdout:669: writing smithi02683976-6 from 7610368 to 8044544 tid 2 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:669: writing smithi02683976-6 from 8044544 to 8413184 tid 3 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:670: copy_from oid 41 from oid 39 current snap is 58 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: finishing write tid 2 to smithi02683976-26 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: finishing write tid 3 to smithi02683976-26 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: finishing write tid 4 to smithi02683976-26 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: finishing write tid 5 to smithi02683976-26 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 147 (ObjNum 233 snap 58 seq_num 233) dirty exists 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: left oid 26 (ObjNum 233 snap 58 seq_num 233) 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:668: done (2 left) 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 58 2024-09-13T14:21:13.995 INFO:tasks.rados.rados.0.smithi026.stdout:671: seq_num 235 ranges {5824512=745472,6569984=712704,7282688=90112} 2024-09-13T14:21:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:671: writing smithi02683976-19 from 5824512 to 6569984 tid 1 2024-09-13T14:21:14.034 INFO:tasks.rados.rados.0.smithi026.stdout:671: writing smithi02683976-19 from 6569984 to 7282688 tid 2 2024-09-13T14:21:14.035 INFO:tasks.rados.rados.0.smithi026.stdout:671: writing smithi02683976-19 from 7282688 to 7372800 tid 3 2024-09-13T14:21:14.035 INFO:tasks.rados.rados.0.smithi026.stdout:672: delete oid 46 current snap is 58 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: finishing write tid 1 to smithi02683976-6 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: finishing write tid 2 to smithi02683976-6 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: finishing write tid 3 to smithi02683976-6 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: finishing write tid 4 to smithi02683976-6 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: finishing write tid 5 to smithi02683976-6 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 370 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:669: left oid 6 (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:670: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:670: finishing copy_from to smithi02683976-41 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 137 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:14.036 INFO:tasks.rados.rados.0.smithi026.stdout:671: finishing write tid 1 to smithi02683976-19 2024-09-13T14:21:14.037 INFO:tasks.rados.rados.0.smithi026.stdout:669: done (3 left) 2024-09-13T14:21:14.037 INFO:tasks.rados.rados.0.smithi026.stdout:670: done (2 left) 2024-09-13T14:21:14.037 INFO:tasks.rados.rados.0.smithi026.stdout:672: done (1 left) 2024-09-13T14:21:14.037 INFO:tasks.rados.rados.0.smithi026.stdout:673: snap_remove snap 57 2024-09-13T14:21:14.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:13 smithi026 ceph-mon[39182]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:21:14.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:13 smithi026 ceph-mon[27770]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:21:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:13 smithi145 ceph-mon[31057]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:21:14.836 INFO:tasks.rados.rados.0.smithi026.stdout:673: done (1 left) 2024-09-13T14:21:14.836 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 58 2024-09-13T14:21:14.837 INFO:tasks.rados.rados.0.smithi026.stdout:671: finishing write tid 2 to smithi02683976-19 2024-09-13T14:21:14.837 INFO:tasks.rados.rados.0.smithi026.stdout:674: seq_num 237 ranges {3792896=778240,4571136=532480,5103616=270336} 2024-09-13T14:21:14.861 INFO:tasks.rados.rados.0.smithi026.stdout:674: writing smithi02683976-36 from 3792896 to 4571136 tid 1 2024-09-13T14:21:14.865 INFO:tasks.rados.rados.0.smithi026.stdout:674: writing smithi02683976-36 from 4571136 to 5103616 tid 2 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:674: writing smithi02683976-36 from 5103616 to 5373952 tid 3 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:675: read oid 17 snap -1 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:675: expect deleted 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:676: snap_create 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:671: finishing write tid 3 to smithi02683976-19 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:671: finishing write tid 4 to smithi02683976-19 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:671: finishing write tid 5 to smithi02683976-19 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 362 (ObjNum 235 snap 58 seq_num 235) dirty exists 2024-09-13T14:21:14.867 INFO:tasks.rados.rados.0.smithi026.stdout:671: left oid 19 (ObjNum 235 snap 58 seq_num 235) 2024-09-13T14:21:14.869 INFO:tasks.rados.rados.0.smithi026.stdout:674: finishing write tid 1 to smithi02683976-36 2024-09-13T14:21:14.869 INFO:tasks.rados.rados.0.smithi026.stdout:674: finishing write tid 2 to smithi02683976-36 2024-09-13T14:21:14.871 INFO:tasks.rados.rados.0.smithi026.stdout:674: finishing write tid 3 to smithi02683976-36 2024-09-13T14:21:14.871 INFO:tasks.rados.rados.0.smithi026.stdout:674: finishing write tid 4 to smithi02683976-36 2024-09-13T14:21:14.873 INFO:tasks.rados.rados.0.smithi026.stdout:674: finishing write tid 5 to smithi02683976-36 2024-09-13T14:21:14.873 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 411 (ObjNum 237 snap 58 seq_num 237) dirty exists 2024-09-13T14:21:14.873 INFO:tasks.rados.rados.0.smithi026.stdout:674: left oid 36 (ObjNum 237 snap 58 seq_num 237) 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:14 smithi026 ceph-mon[39182]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:14 smithi026 ceph-mon[39182]: pgmap v45: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:14 smithi026 ceph-mon[27770]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:14 smithi026 ceph-mon[27770]: pgmap v45: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:15.071 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:14 smithi026 systemd[1]: Stopping Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:14 smithi145 ceph-mon[31057]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:21:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:14 smithi145 ceph-mon[31057]: pgmap v45: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:15.410 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[79182]: ts=2024-09-13T14:21:15.069Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:21:15.410 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 podman[86890]: 2024-09-13 14:21:15.121685142 +0000 UTC m=+0.176199133 container died 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:15.665 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 podman[86890]: 2024-09-13 14:21:15.509372149 +0000 UTC m=+0.563886140 container cleanup 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:15.665 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 bash[86890]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:671: done (3 left) 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:674: done (2 left) 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:675: done (1 left) 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:676: done (0 left) 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:677: setattr oid 33 current snap is 59 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:678: read oid 6 snap -1 2024-09-13T14:21:15.842 INFO:tasks.rados.rados.0.smithi026.stdout:678: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:15.843 INFO:tasks.rados.rados.0.smithi026.stdout:679: delete oid 27 current snap is 59 2024-09-13T14:21:15.846 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 412 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:21:15.847 INFO:tasks.rados.rados.0.smithi026.stdout:677: done (2 left) 2024-09-13T14:21:15.847 INFO:tasks.rados.rados.0.smithi026.stdout:679: done (1 left) 2024-09-13T14:21:15.847 INFO:tasks.rados.rados.0.smithi026.stdout:680: rollback oid 34 current snap is 59 2024-09-13T14:21:15.847 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 48 2024-09-13T14:21:15.848 INFO:tasks.rados.rados.0.smithi026.stdout:681: read oid 9 snap -1 2024-09-13T14:21:15.848 INFO:tasks.rados.rados.0.smithi026.stdout:681: expect (ObjNum 161 snap 38 seq_num 161) 2024-09-13T14:21:15.848 INFO:tasks.rados.rados.0.smithi026.stdout:682: read oid 18 snap 58 2024-09-13T14:21:15.848 INFO:tasks.rados.rados.0.smithi026.stdout:682: expect (ObjNum 104 snap 17 seq_num 104) 2024-09-13T14:21:15.848 INFO:tasks.rados.rados.0.smithi026.stdout:683: snap_create 2024-09-13T14:21:15.854 INFO:tasks.rados.rados.0.smithi026.stdout:680: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:21:15.854 INFO:tasks.rados.rados.0.smithi026.stdout:680: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:21:15.854 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 233 (ObjNum 129 snap 0 seq_num 519199248) dirty dne 2024-09-13T14:21:15.857 INFO:tasks.rados.rados.0.smithi026.stdout:682: expect (ObjNum 104 snap 17 seq_num 104) 2024-09-13T14:21:15.993 INFO:tasks.rados.rados.0.smithi026.stdout:678: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:16.027 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:15 smithi026 ceph-mon[27770]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:21:16.027 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:15 smithi026 ceph-mon[39182]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:21:16.027 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 podman[86909]: 2024-09-13 14:21:15.76923462 +0000 UTC m=+0.647003757 container remove 52eac4b9b3e58385acdb1f66ce307d15f57b9adfc6828ed3019d3fbaf1813015 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:16.028 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:15 smithi026 podman[86909]: 2024-09-13 14:21:15.844814801 +0000 UTC m=+0.722583941 volume remove 4e9b57ae05541b8d878c090dcc2766befb29fa949f97d71f6d331ca37f5719ac 2024-09-13T14:21:16.028 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:21:16.028 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 systemd[1]: Stopped Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:16.028 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.136s CPU time. 2024-09-13T14:21:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:15 smithi145 ceph-mon[31057]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:21:16.415 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 systemd[1]: Starting Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:16.589 INFO:tasks.rados.rados.0.smithi026.stdout:681: expect (ObjNum 161 snap 38 seq_num 161) 2024-09-13T14:21:16.843 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 podman[87038]: 2024-09-13 14:21:16.429401958 +0000 UTC m=+0.103387833 volume create 8778078b5afbe3bdb7db8aae304013a3e25a6e7f2b7cf7538dfe2fb189edcdef 2024-09-13T14:21:16.844 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 podman[87038]: 2024-09-13 14:21:16.342459643 +0000 UTC m=+0.016445530 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-13T14:21:16.844 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:16 smithi026 podman[87038]: 2024-09-13 14:21:16.521591393 +0000 UTC m=+0.195577273 container create 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:16.847 INFO:tasks.rados.rados.0.smithi026.stdout:678: done (4 left) 2024-09-13T14:21:16.848 INFO:tasks.rados.rados.0.smithi026.stdout:680: done (3 left) 2024-09-13T14:21:16.848 INFO:tasks.rados.rados.0.smithi026.stdout:681: done (2 left) 2024-09-13T14:21:16.848 INFO:tasks.rados.rados.0.smithi026.stdout:682: done (1 left) 2024-09-13T14:21:16.848 INFO:tasks.rados.rados.0.smithi026.stdout:683: done (0 left) 2024-09-13T14:21:16.848 INFO:tasks.rados.rados.0.smithi026.stdout:684: snap_remove snap 59 2024-09-13T14:21:17.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[39182]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:21:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[39182]: pgmap v48: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[27770]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:21:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[27770]: pgmap v48: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:16 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:16 smithi145 ceph-mon[31057]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:21:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:16 smithi145 ceph-mon[31057]: pgmap v48: 121 pgs: 121 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-13T14:21:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:16 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:17.665 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 podman[87038]: 2024-09-13 14:21:17.402039662 +0000 UTC m=+1.076025538 container init 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 podman[87038]: 2024-09-13 14:21:17.408211938 +0000 UTC m=+1.082197816 container start 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.420Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.420Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.422Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.423Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.447Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.447Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.449Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:17.449Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:21:17.666 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 bash[87038]: 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 2024-09-13T14:21:17.667 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:17 smithi026 systemd[1]: Started Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:17.852 INFO:tasks.rados.rados.0.smithi026.stdout:684: done (0 left) 2024-09-13T14:21:17.852 INFO:tasks.rados.rados.0.smithi026.stdout:685: snap_remove snap 58 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[39182]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[39182]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[39182]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[27770]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[27770]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:21:18.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:17 smithi026 ceph-mon[27770]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:21:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:17 smithi145 ceph-mon[31057]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:21:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:17 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:17 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:17 smithi145 ceph-mon[31057]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:21:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:17 smithi145 ceph-mon[31057]: Deploying daemon node-exporter.a on smithi026 2024-09-13T14:21:18.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:18 smithi026 systemd[1]: Stopping Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:18.725 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:18 smithi026 podman[87236]: 2024-09-13 14:21:18.42006148 +0000 UTC m=+0.139874437 container died 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:18.749 INFO:tasks.rados.rados.0.smithi026.stdout:685: done (0 left) 2024-09-13T14:21:18.749 INFO:tasks.rados.rados.0.smithi026.stdout:686: copy_from oid 48 from oid 5 current snap is 60 2024-09-13T14:21:18.749 INFO:tasks.rados.rados.0.smithi026.stdout:687: setattr oid 20 current snap is 60 2024-09-13T14:21:18.750 INFO:tasks.rados.rados.0.smithi026.stdout:686: finishing copy_from to smithi02683976-48 2024-09-13T14:21:18.750 INFO:tasks.rados.rados.0.smithi026.stdout:686: got expected ENOENT (src dne) 2024-09-13T14:21:18.750 INFO:tasks.rados.rados.0.smithi026.stdout:686: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:21:18.750 INFO:tasks.rados.rados.0.smithi026.stdout:686: done (1 left) 2024-09-13T14:21:18.750 INFO:tasks.rados.rados.0.smithi026.stdout:688: read oid 8 snap -1 2024-09-13T14:21:18.751 INFO:tasks.rados.rados.0.smithi026.stdout:688: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:18.751 INFO:tasks.rados.rados.0.smithi026.stdout:689: snap_remove snap 54 2024-09-13T14:21:19.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:18 smithi026 podman[87236]: 2024-09-13 14:21:18.891077195 +0000 UTC m=+0.610890144 container cleanup 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:19.151 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:18 smithi026 bash[87236]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:21:19.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[27770]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:21:19.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[27770]: pgmap v51: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:19.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[27770]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:21:19.152 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[39182]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:21:19.152 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[39182]: pgmap v51: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:19.152 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:18 smithi026 ceph-mon[39182]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:21:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:18 smithi145 ceph-mon[31057]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:21:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:18 smithi145 ceph-mon[31057]: pgmap v51: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:18 smithi145 ceph-mon[31057]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:21:19.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 podman[87254]: 2024-09-13 14:21:19.15059759 +0000 UTC m=+0.734263556 container remove 0720764634c2a0fbfc3559285c626341baf84b0eb95fdb7a38b59049786b2fe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:19.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:21:19.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:21:19.416 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 systemd[1]: Stopped Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:19.416 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.378s CPU time. 2024-09-13T14:21:19.714 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 systemd[1]: Starting Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:19.715 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:19.424Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000381112s 2024-09-13T14:21:19.748 INFO:tasks.rados.rados.0.smithi026.stdout:689: done (2 left) 2024-09-13T14:21:19.748 INFO:tasks.rados.rados.0.smithi026.stdout:690: rmattr oid 48 current snap is 60 2024-09-13T14:21:19.748 INFO:tasks.rados.rados.0.smithi026.stdout:690: done (2 left) 2024-09-13T14:21:19.748 INFO:tasks.rados.rados.0.smithi026.stdout:691: delete oid 2 current snap is 60 2024-09-13T14:21:19.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 366 (ObjNum 129 snap 0 seq_num 519592984) dirty exists 2024-09-13T14:21:19.749 INFO:tasks.rados.rados.0.smithi026.stdout:688: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:19.774 INFO:tasks.rados.rados.0.smithi026.stdout:687: done (2 left) 2024-09-13T14:21:19.774 INFO:tasks.rados.rados.0.smithi026.stdout:688: done (1 left) 2024-09-13T14:21:19.774 INFO:tasks.rados.rados.0.smithi026.stdout:691: done (0 left) 2024-09-13T14:21:19.774 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 60 2024-09-13T14:21:19.774 INFO:tasks.rados.rados.0.smithi026.stdout:692: seq_num 240 ranges {3137536=425984,3563520=589824,4153344=483328,4636672=131072} 2024-09-13T14:21:19.793 INFO:tasks.rados.rados.0.smithi026.stdout:692: writing smithi02683976-4 from 3137536 to 3563520 tid 1 2024-09-13T14:21:19.796 INFO:tasks.rados.rados.0.smithi026.stdout:692: writing smithi02683976-4 from 3563520 to 4153344 tid 2 2024-09-13T14:21:19.799 INFO:tasks.rados.rados.0.smithi026.stdout:692: writing smithi02683976-4 from 4153344 to 4636672 tid 3 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:692: writing smithi02683976-4 from 4636672 to 4767744 tid 4 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:693: read oid 11 snap -1 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:693: expect (ObjNum 196 snap 46 seq_num 196) 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:694: read oid 42 snap 53 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:694: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:695: setattr oid 18 current snap is 60 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 1 to smithi02683976-4 2024-09-13T14:21:19.801 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 2 to smithi02683976-4 2024-09-13T14:21:19.804 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 3 to smithi02683976-4 2024-09-13T14:21:19.804 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 4 to smithi02683976-4 2024-09-13T14:21:19.804 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 5 to smithi02683976-4 2024-09-13T14:21:19.805 INFO:tasks.rados.rados.0.smithi026.stdout:696: snap_remove snap 48 2024-09-13T14:21:20.033 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:19 smithi026 bash[87388]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:21:20.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:20 smithi026 bash[87388]: Getting image source signatures 2024-09-13T14:21:20.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:20 smithi026 bash[87388]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:21:20.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:20 smithi026 bash[87388]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:21:20.415 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:20 smithi026 bash[87388]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:21:20.755 INFO:tasks.rados.rados.0.smithi026.stdout:696: done (4 left) 2024-09-13T14:21:20.755 INFO:tasks.rados.rados.0.smithi026.stdout:697: delete oid 15 current snap is 60 2024-09-13T14:21:20.755 INFO:tasks.rados.rados.0.smithi026.stdout:693: expect (ObjNum 196 snap 46 seq_num 196) 2024-09-13T14:21:20.792 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 306 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:21:20.793 INFO:tasks.rados.rados.0.smithi026.stdout:692: finishing write tid 6 to smithi02683976-4 2024-09-13T14:21:20.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 174 (ObjNum 240 snap 60 seq_num 240) dirty exists 2024-09-13T14:21:20.793 INFO:tasks.rados.rados.0.smithi026.stdout:692: left oid 4 (ObjNum 240 snap 60 seq_num 240) 2024-09-13T14:21:20.793 INFO:tasks.rados.rados.0.smithi026.stdout:694: expect (ObjNum 116 snap 24 seq_num 116) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:692: done (4 left) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:693: done (3 left) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:694: done (2 left) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:695: done (1 left) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:697: done (0 left) 2024-09-13T14:21:20.948 INFO:tasks.rados.rados.0.smithi026.stdout:698: snap_create 2024-09-13T14:21:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:20 smithi026 ceph-mon[39182]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:21:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:20 smithi026 ceph-mon[39182]: pgmap v54: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:20 smithi026 ceph-mon[27770]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:21:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:20 smithi026 ceph-mon[27770]: pgmap v54: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:20 smithi145 ceph-mon[31057]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:21:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:20 smithi145 ceph-mon[31057]: pgmap v54: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-13T14:21:21.761 INFO:tasks.rados.rados.0.smithi026.stdout:698: done (0 left) 2024-09-13T14:21:21.761 INFO:tasks.rados.rados.0.smithi026.stdout:699: setattr oid 43 current snap is 61 2024-09-13T14:21:21.762 INFO:tasks.rados.rados.0.smithi026.stdout:700: snap_remove snap 53 2024-09-13T14:21:21.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:21 smithi026 ceph-mon[27770]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:21:21.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:21 smithi026 ceph-mon[39182]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:21:21.916 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:21 smithi026 bash[87388]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:21:21.916 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:21 smithi026 bash[87388]: Writing manifest to image destination 2024-09-13T14:21:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:21 smithi145 ceph-mon[31057]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:21:22.348 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 podman[87388]: 2024-09-13 14:21:21.997302179 +0000 UTC m=+2.377671907 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:21:22.348 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 podman[87388]: 2024-09-13 14:21:22.087162808 +0000 UTC m=+2.467532532 container create 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:22.666 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 podman[87388]: 2024-09-13 14:21:22.347948816 +0000 UTC m=+2.728318533 container init 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:22.666 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 podman[87388]: 2024-09-13 14:21:22.354542098 +0000 UTC m=+2.734911809 container start 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.358Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:21:22.667 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:21:22.668 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:21:22.669 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:21:22.670 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.359Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.360Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:21:22.671 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a[87463]: ts=2024-09-13T14:21:22.360Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:21:22.672 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 bash[87388]: 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc 2024-09-13T14:21:22.672 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:21:22 smithi026 systemd[1]: Started Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:700: done (1 left) 2024-09-13T14:21:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:701: read oid 21 snap -1 2024-09-13T14:21:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:701: expect (ObjNum 226 snap 55 seq_num 226) 2024-09-13T14:21:22.769 INFO:tasks.rados.rados.0.smithi026.stdout:702: snap_remove snap 60 2024-09-13T14:21:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[27770]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:21:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[27770]: pgmap v57: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[27770]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[39182]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[39182]: pgmap v57: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:22 smithi026 ceph-mon[39182]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:22] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.43.0" 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-mon[31057]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-mon[31057]: pgmap v57: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:22 smithi145 ceph-mon[31057]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:21:23.424 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:702: done (2 left) 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:703: copy_from oid 46 from oid 26 current snap is 61 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:704: copy_from oid 44 from oid 42 current snap is 61 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 384 (ObjNum 186 snap 44 seq_num 186) dirty exists 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:699: done (3 left) 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:705: setattr oid 16 current snap is 61 2024-09-13T14:21:23.768 INFO:tasks.rados.rados.0.smithi026.stdout:701: expect (ObjNum 226 snap 55 seq_num 226) 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:703: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:704: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:704: finishing copy_from to smithi02683976-44 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 237 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:703: finishing copy_from to smithi02683976-46 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 147 (ObjNum 233 snap 58 seq_num 233) dirty exists 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:701: done (3 left) 2024-09-13T14:21:23.811 INFO:tasks.rados.rados.0.smithi026.stdout:703: done (2 left) 2024-09-13T14:21:23.812 INFO:tasks.rados.rados.0.smithi026.stdout:704: done (1 left) 2024-09-13T14:21:23.812 INFO:tasks.rados.rados.0.smithi026.stdout:706: rollback oid 13 current snap is 61 2024-09-13T14:21:23.812 INFO:tasks.rados.rados.0.smithi026.stdout:706: done (1 left) 2024-09-13T14:21:23.812 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 61 2024-09-13T14:21:23.812 INFO:tasks.rados.rados.0.smithi026.stdout:707: seq_num 242 ranges {3252224=458752,3710976=753664,4464640=450560} 2024-09-13T14:21:23.831 INFO:tasks.rados.rados.0.smithi026.stdout:707: writing smithi02683976-32 from 3252224 to 3710976 tid 1 2024-09-13T14:21:23.835 INFO:tasks.rados.rados.0.smithi026.stdout:707: writing smithi02683976-32 from 3710976 to 4464640 tid 2 2024-09-13T14:21:23.839 INFO:tasks.rados.rados.0.smithi026.stdout:707: writing smithi02683976-32 from 4464640 to 4915200 tid 3 2024-09-13T14:21:23.839 INFO:tasks.rados.rados.0.smithi026.stdout:708: copy_from oid 43 from oid 2 current snap is 61 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 397 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:705: done (2 left) 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:709: delete oid 35 current snap is 61 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:707: finishing write tid 1 to smithi02683976-32 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:707: finishing write tid 2 to smithi02683976-32 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:708: finishing copy_from to smithi02683976-43 2024-09-13T14:21:23.840 INFO:tasks.rados.rados.0.smithi026.stdout:708: got expected ENOENT (src dne) 2024-09-13T14:21:23.841 INFO:tasks.rados.rados.0.smithi026.stdout:708: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:21:23.841 INFO:tasks.rados.rados.0.smithi026.stdout:708: done (2 left) 2024-09-13T14:21:23.841 INFO:tasks.rados.rados.0.smithi026.stdout:709: done (1 left) 2024-09-13T14:21:23.841 INFO:tasks.rados.rados.0.smithi026.stdout:710: read oid 28 snap -1 2024-09-13T14:21:23.841 INFO:tasks.rados.rados.0.smithi026.stdout:710: expect (ObjNum 213 snap 53 seq_num 213) 2024-09-13T14:21:23.842 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 61 2024-09-13T14:21:23.842 INFO:tasks.rados.rados.0.smithi026.stdout:711: seq_num 244 ranges {1425408=491520,1916928=573440,2490368=458752,2949120=172032} 2024-09-13T14:21:23.851 INFO:tasks.rados.rados.0.smithi026.stdout:711: writing smithi02683976-44 from 1425408 to 1916928 tid 1 2024-09-13T14:21:23.855 INFO:tasks.rados.rados.0.smithi026.stdout:711: writing smithi02683976-44 from 1916928 to 2490368 tid 2 2024-09-13T14:21:23.858 INFO:tasks.rados.rados.0.smithi026.stdout:711: writing smithi02683976-44 from 2490368 to 2949120 tid 3 2024-09-13T14:21:23.860 INFO:tasks.rados.rados.0.smithi026.stdout:711: writing smithi02683976-44 from 2949120 to 3121152 tid 4 2024-09-13T14:21:23.860 INFO:tasks.rados.rados.0.smithi026.stdout:712: read oid 30 snap -1 2024-09-13T14:21:23.860 INFO:tasks.rados.rados.0.smithi026.stdout:712: expect (ObjNum 164 snap 38 seq_num 164) 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:713: read oid 49 snap -1 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:713: expect (ObjNum 199 snap 47 seq_num 199) 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:714: rmattr oid 18 current snap is 61 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:707: finishing write tid 3 to smithi02683976-32 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:715: rollback oid 16 current snap is 61 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:715: done (6 left) 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:716: copy_from oid 13 from oid 37 current snap is 61 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:707: finishing write tid 4 to smithi02683976-32 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:707: finishing write tid 5 to smithi02683976-32 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 322 (ObjNum 242 snap 61 seq_num 242) dirty exists 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:707: left oid 32 (ObjNum 242 snap 61 seq_num 242) 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 1 to smithi02683976-44 2024-09-13T14:21:23.861 INFO:tasks.rados.rados.0.smithi026.stdout:710: expect (ObjNum 213 snap 53 seq_num 213) 2024-09-13T14:21:24.036 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 starting - - - - 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (4m) 30s ago 4m 88.7M - 9.4.7 954c08fa6188 b034aa056559 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (6m) 32s ago 6m 13.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (6m) 30s ago 6m 19.1M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283 running (46s) 30s ago 9m 495M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4ae2b625819 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283,8765 running (11m) 32s ago 11m 274M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 90a812e595d2 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (11m) 32s ago 11m 56.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (10m) 30s ago 10m 47.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 32s ago 9m 46.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 starting - - - - 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (4m) 30s ago 4m 15.0M - 1.5.0 0da6a335fe13 02ae27c8e5f5 2024-09-13T14:21:24.037 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (9m) 32s ago 9m 112M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) 32s ago 8m 99.6M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (8m) 32s ago 8m 107M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (8m) 32s ago 8m 67.3M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (7m) 30s ago 7m 119M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (7m) 30s ago 7m 99.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (6m) 30s ago 6m 73.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (6m) 30s ago 6m 76.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:21:24.038 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (3m) 30s ago 5m 54.0M - 2.43.0 a07b618ecd1d 33771513b4fc 2024-09-13T14:21:24.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[39182]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:21:24.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[39182]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:21:24.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[39182]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[39182]: from='client.15102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[27770]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:21:24.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[27770]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:21:24.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[27770]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:23 smithi026 ceph-mon[27770]: from='client.15102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 2 to smithi02683976-44 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 3 to smithi02683976-44 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 4 to smithi02683976-44 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 5 to smithi02683976-44 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 323 (ObjNum 104 snap 17 seq_num 104) dirty exists 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: finishing write tid 6 to smithi02683976-44 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 242 (ObjNum 244 snap 61 seq_num 244) dirty exists 2024-09-13T14:21:24.090 INFO:tasks.rados.rados.0.smithi026.stdout:711: left oid 44 (ObjNum 244 snap 61 seq_num 244) 2024-09-13T14:21:24.091 INFO:tasks.rados.rados.0.smithi026.stdout:716: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:21:24.091 INFO:tasks.rados.rados.0.smithi026.stdout:716: finishing copy_from to smithi02683976-13 2024-09-13T14:21:24.091 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 232 (ObjNum 144 snap 31 seq_num 144) dirty exists 2024-09-13T14:21:24.091 INFO:tasks.rados.rados.0.smithi026.stdout:713: expect (ObjNum 199 snap 47 seq_num 199) 2024-09-13T14:21:24.115 INFO:tasks.rados.rados.0.smithi026.stdout:712: expect (ObjNum 164 snap 38 seq_num 164) 2024-09-13T14:21:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:23 smithi145 ceph-mon[31057]: Deploying daemon prometheus.a on smithi145 2024-09-13T14:21:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:23 smithi145 ceph-mon[31057]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:21:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:23 smithi145 ceph-mon[31057]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:23 smithi145 ceph-mon[31057]: from='client.15102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:24.381 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:21:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 14, 2024-09-13T14:21:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:21:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:21:24.383 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:21:24.428 INFO:tasks.rados.rados.0.smithi026.stdout:707: done (6 left) 2024-09-13T14:21:24.428 INFO:tasks.rados.rados.0.smithi026.stdout:710: done (5 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:711: done (4 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:712: done (3 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:713: done (2 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:714: done (1 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:716: done (0 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:717: rollback oid 11 current snap is 61 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:717: done (0 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:718: rmattr oid 13 current snap is 61 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:718: done (0 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:719: copy_from oid 32 from oid 6 current snap is 61 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:720: rollback oid 1 current snap is 61 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:720: done (1 left) 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 61 2024-09-13T14:21:24.429 INFO:tasks.rados.rados.0.smithi026.stdout:721: seq_num 245 ranges {3612672=507904,4120576=458752,4579328=671744,5251072=73728} 2024-09-13T14:21:24.452 INFO:tasks.rados.rados.0.smithi026.stdout:721: writing smithi02683976-43 from 3612672 to 4120576 tid 1 2024-09-13T14:21:24.455 INFO:tasks.rados.rados.0.smithi026.stdout:721: writing smithi02683976-43 from 4120576 to 4579328 tid 2 2024-09-13T14:21:24.460 INFO:tasks.rados.rados.0.smithi026.stdout:721: writing smithi02683976-43 from 4579328 to 5251072 tid 3 2024-09-13T14:21:24.460 INFO:tasks.rados.rados.0.smithi026.stdout:721: writing smithi02683976-43 from 5251072 to 5324800 tid 4 2024-09-13T14:21:24.461 INFO:tasks.rados.rados.0.smithi026.stdout:722: delete oid 30 current snap is 61 2024-09-13T14:21:24.461 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 1 to smithi02683976-43 2024-09-13T14:21:24.462 INFO:tasks.rados.rados.0.smithi026.stdout:719: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:21:24.463 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 2 to smithi02683976-43 2024-09-13T14:21:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:722: done (2 left) 2024-09-13T14:21:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 61 2024-09-13T14:21:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:723: seq_num 247 ranges {16384=540672,557056=622592,1179648=581632} 2024-09-13T14:21:24.467 INFO:tasks.rados.rados.0.smithi026.stdout:723: writing smithi02683976-13 from 16384 to 557056 tid 1 2024-09-13T14:21:24.471 INFO:tasks.rados.rados.0.smithi026.stdout:723: writing smithi02683976-13 from 557056 to 1179648 tid 2 2024-09-13T14:21:24.475 INFO:tasks.rados.rados.0.smithi026.stdout:723: writing smithi02683976-13 from 1179648 to 1761280 tid 3 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:724: delete oid 4 current snap is 61 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 3 to smithi02683976-43 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 4 to smithi02683976-43 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:719: finishing copy_from to smithi02683976-32 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 370 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 5 to smithi02683976-43 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:723: finishing write tid 1 to smithi02683976-13 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:721: finishing write tid 6 to smithi02683976-43 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 403 (ObjNum 245 snap 61 seq_num 245) dirty exists 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:721: left oid 43 (ObjNum 245 snap 61 seq_num 245) 2024-09-13T14:21:24.476 INFO:tasks.rados.rados.0.smithi026.stdout:723: finishing write tid 2 to smithi02683976-13 2024-09-13T14:21:24.477 INFO:tasks.rados.rados.0.smithi026.stdout:719: done (3 left) 2024-09-13T14:21:24.477 INFO:tasks.rados.rados.0.smithi026.stdout:721: done (2 left) 2024-09-13T14:21:24.477 INFO:tasks.rados.rados.0.smithi026.stdout:724: done (1 left) 2024-09-13T14:21:24.477 INFO:tasks.rados.rados.0.smithi026.stdout:725: snap_create 2024-09-13T14:21:24.478 INFO:tasks.rados.rados.0.smithi026.stdout:723: finishing write tid 3 to smithi02683976-13 2024-09-13T14:21:24.478 INFO:tasks.rados.rados.0.smithi026.stdout:723: finishing write tid 4 to smithi02683976-13 2024-09-13T14:21:24.479 INFO:tasks.rados.rados.0.smithi026.stdout:723: finishing write tid 5 to smithi02683976-13 2024-09-13T14:21:24.479 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 407 (ObjNum 247 snap 61 seq_num 247) dirty exists 2024-09-13T14:21:24.479 INFO:tasks.rados.rados.0.smithi026.stdout:723: left oid 13 (ObjNum 247 snap 61 seq_num 247) 2024-09-13T14:21:24.688 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:21:24.688 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/20 daemons upgraded", 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:21:24.689 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:21:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:723: done (1 left) 2024-09-13T14:21:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:725: done (0 left) 2024-09-13T14:21:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:726: rmattr oid 13 current snap is 62 2024-09-13T14:21:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:726: done (0 left) 2024-09-13T14:21:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:727: snap_create 2024-09-13T14:21:25.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[39182]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:21:25.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[39182]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[39182]: pgmap v60: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1462937731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[39182]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[27770]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[27770]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[27770]: pgmap v60: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1462937731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:25.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:24 smithi026 ceph-mon[27770]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:24 smithi145 ceph-mon[31057]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:21:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:24 smithi145 ceph-mon[31057]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:24 smithi145 ceph-mon[31057]: pgmap v60: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:24 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1462937731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:24 smithi145 ceph-mon[31057]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:25 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[28022]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:25.779 INFO:tasks.rados.rados.0.smithi026.stdout:727: done (0 left) 2024-09-13T14:21:25.779 INFO:tasks.rados.rados.0.smithi026.stdout:728: copy_from oid 43 from oid 39 current snap is 63 2024-09-13T14:21:25.779 INFO:tasks.rados.rados.0.smithi026.stdout:729: delete oid 49 current snap is 63 2024-09-13T14:21:25.787 INFO:tasks.rados.rados.0.smithi026.stdout:729: done (1 left) 2024-09-13T14:21:25.787 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 63 2024-09-13T14:21:25.788 INFO:tasks.rados.rados.0.smithi026.stdout:730: seq_num 250 ranges {0=589824,589824=679936,1269760=409600,1679360=114688} 2024-09-13T14:21:25.791 INFO:tasks.rados.rados.0.smithi026.stdout:730: writing smithi02683976-23 from 0 to 589824 tid 1 2024-09-13T14:21:25.796 INFO:tasks.rados.rados.0.smithi026.stdout:730: writing smithi02683976-23 from 589824 to 1269760 tid 2 2024-09-13T14:21:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:730: writing smithi02683976-23 from 1269760 to 1679360 tid 3 2024-09-13T14:21:25.799 INFO:tasks.rados.rados.0.smithi026.stdout:730: writing smithi02683976-23 from 1679360 to 1794048 tid 4 2024-09-13T14:21:25.799 INFO:tasks.rados.rados.0.smithi026.stdout:731: rollback oid 3 current snap is 63 2024-09-13T14:21:25.799 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 62 2024-09-13T14:21:25.799 INFO:tasks.rados.rados.0.smithi026.stdout:728: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:728: finishing copy_from to smithi02683976-43 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 137 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 1 to smithi02683976-23 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:728: done (2 left) 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:732: snap_create 2024-09-13T14:21:25.800 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 2 to smithi02683976-23 2024-09-13T14:21:25.802 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 3 to smithi02683976-23 2024-09-13T14:21:25.802 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 4 to smithi02683976-23 2024-09-13T14:21:25.803 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 5 to smithi02683976-23 2024-09-13T14:21:25.803 INFO:tasks.rados.rados.0.smithi026.stdout:731: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:21:25.803 INFO:tasks.rados.rados.0.smithi026.stdout:731: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:21:25.803 INFO:tasks.rados.rados.0.smithi026.stdout:731: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:21:25.804 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 438 (ObjNum 224 snap 55 seq_num 224) dirty exists 2024-09-13T14:21:25.804 INFO:tasks.rados.rados.0.smithi026.stdout:730: finishing write tid 6 to smithi02683976-23 2024-09-13T14:21:25.804 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 375 (ObjNum 250 snap 63 seq_num 250) dirty exists 2024-09-13T14:21:25.804 INFO:tasks.rados.rados.0.smithi026.stdout:730: left oid 23 (ObjNum 250 snap 63 seq_num 250) 2024-09-13T14:21:26.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:25 smithi026 ceph-mon[27770]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:21:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:25 smithi026 ceph-mon[39182]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:21:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:25 smithi145 ceph-mon[31057]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:730: done (2 left) 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:731: done (1 left) 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:732: done (0 left) 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:733: read oid 42 snap -1 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:733: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:734: read oid 24 snap -1 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:734: expect (ObjNum 231 snap 57 seq_num 231) 2024-09-13T14:21:26.791 INFO:tasks.rados.rados.0.smithi026.stdout:735: snap_create 2024-09-13T14:21:26.798 INFO:tasks.rados.rados.0.smithi026.stdout:733: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:21:26.836 INFO:tasks.rados.rados.0.smithi026.stdout:734: expect (ObjNum 231 snap 57 seq_num 231) 2024-09-13T14:21:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:26 smithi026 ceph-mon[27770]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:21:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:26 smithi026 ceph-mon[27770]: pgmap v63: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:26 smithi026 ceph-mon[39182]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:21:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:26 smithi026 ceph-mon[39182]: pgmap v63: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:26 smithi145 ceph-mon[31057]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:21:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:26 smithi145 ceph-mon[31057]: pgmap v63: 121 pgs: 121 active+clean; 111 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 398 KiB/s wr, 19 op/s 2024-09-13T14:21:27.791 INFO:tasks.rados.rados.0.smithi026.stdout:733: done (2 left) 2024-09-13T14:21:27.791 INFO:tasks.rados.rados.0.smithi026.stdout:734: done (1 left) 2024-09-13T14:21:27.791 INFO:tasks.rados.rados.0.smithi026.stdout:735: done (0 left) 2024-09-13T14:21:27.791 INFO:tasks.rados.rados.0.smithi026.stdout:736: delete oid 10 current snap is 65 2024-09-13T14:21:27.792 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:21:27 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:21:27.427Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003573091s 2024-09-13T14:21:27.794 INFO:tasks.rados.rados.0.smithi026.stdout:736: done (0 left) 2024-09-13T14:21:27.794 INFO:tasks.rados.rados.0.smithi026.stdout:737: snap_create 2024-09-13T14:21:28.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:27 smithi026 ceph-mon[27770]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:21:28.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:27 smithi026 ceph-mon[39182]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:21:28.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:27 smithi145 ceph-mon[31057]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:737: done (0 left) 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:738: read oid 17 snap -1 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:738: expect deleted 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:739: rollback oid 6 current snap is 66 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 65 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:740: read oid 27 snap 64 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:740: expect deleted 2024-09-13T14:21:28.752 INFO:tasks.rados.rados.0.smithi026.stdout:741: copy_from oid 18 from oid 22 current snap is 66 2024-09-13T14:21:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:742: rmattr oid 14 current snap is 66 2024-09-13T14:21:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:742: done (4 left) 2024-09-13T14:21:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 66 2024-09-13T14:21:28.753 INFO:tasks.rados.rados.0.smithi026.stdout:743: seq_num 252 ranges {1064960=630784,1695744=499712,2195456=425984,2621440=278528} 2024-09-13T14:21:28.761 INFO:tasks.rados.rados.0.smithi026.stdout:743: writing smithi02683976-11 from 1064960 to 1695744 tid 1 2024-09-13T14:21:28.764 INFO:tasks.rados.rados.0.smithi026.stdout:743: writing smithi02683976-11 from 1695744 to 2195456 tid 2 2024-09-13T14:21:28.766 INFO:tasks.rados.rados.0.smithi026.stdout:743: writing smithi02683976-11 from 2195456 to 2621440 tid 3 2024-09-13T14:21:28.768 INFO:tasks.rados.rados.0.smithi026.stdout:743: writing smithi02683976-11 from 2621440 to 2899968 tid 4 2024-09-13T14:21:28.768 INFO:tasks.rados.rados.0.smithi026.stdout:744: setattr oid 25 current snap is 66 2024-09-13T14:21:28.768 INFO:tasks.rados.rados.0.smithi026.stdout:739: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:21:28.768 INFO:tasks.rados.rados.0.smithi026.stdout:739: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:21:28.768 INFO:tasks.rados.rados.0.smithi026.stdout:739: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:21:28.769 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 394 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:28.769 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 1 to smithi02683976-11 2024-09-13T14:21:28.769 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 2 to smithi02683976-11 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:738: done (5 left) 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:739: done (4 left) 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:740: done (3 left) 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:745: read oid 8 snap -1 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:745: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 66 2024-09-13T14:21:28.770 INFO:tasks.rados.rados.0.smithi026.stdout:746: seq_num 253 ranges {0=647168,647168=786432,1433600=417792} 2024-09-13T14:21:28.773 INFO:tasks.rados.rados.0.smithi026.stdout:746: writing smithi02683976-49 from 0 to 647168 tid 1 2024-09-13T14:21:28.778 INFO:tasks.rados.rados.0.smithi026.stdout:746: writing smithi02683976-49 from 647168 to 1433600 tid 2 2024-09-13T14:21:28.781 INFO:tasks.rados.rados.0.smithi026.stdout:746: writing smithi02683976-49 from 1433600 to 1851392 tid 3 2024-09-13T14:21:28.781 INFO:tasks.rados.rados.0.smithi026.stdout:747: delete oid 35 current snap is 66 2024-09-13T14:21:28.781 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 3 to smithi02683976-11 2024-09-13T14:21:28.781 INFO:tasks.rados.rados.0.smithi026.stdout:741: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 4 to smithi02683976-11 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 5 to smithi02683976-11 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:741: finishing copy_from to smithi02683976-18 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 76 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:743: finishing write tid 6 to smithi02683976-11 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 400 (ObjNum 252 snap 66 seq_num 252) dirty exists 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:743: left oid 11 (ObjNum 252 snap 66 seq_num 252) 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 198 (ObjNum 218 snap 53 seq_num 218) dirty exists 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:746: finishing write tid 1 to smithi02683976-49 2024-09-13T14:21:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:745: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-13T14:21:28.806 INFO:tasks.rados.rados.0.smithi026.stdout:746: finishing write tid 2 to smithi02683976-49 2024-09-13T14:21:28.807 INFO:tasks.rados.rados.0.smithi026.stdout:746: finishing write tid 3 to smithi02683976-49 2024-09-13T14:21:28.807 INFO:tasks.rados.rados.0.smithi026.stdout:746: finishing write tid 4 to smithi02683976-49 2024-09-13T14:21:28.807 INFO:tasks.rados.rados.0.smithi026.stdout:746: finishing write tid 5 to smithi02683976-49 2024-09-13T14:21:28.807 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 228 (ObjNum 253 snap 66 seq_num 253) dirty exists 2024-09-13T14:21:28.807 INFO:tasks.rados.rados.0.smithi026.stdout:746: left oid 49 (ObjNum 253 snap 66 seq_num 253) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:741: done (5 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:743: done (4 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:744: done (3 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:745: done (2 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:746: done (1 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:747: done (0 left) 2024-09-13T14:21:28.808 INFO:tasks.rados.rados.0.smithi026.stdout:748: read oid 18 snap -1 2024-09-13T14:21:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:748: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:21:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:749: read oid 42 snap 62 2024-09-13T14:21:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:749: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:21:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 66 2024-09-13T14:21:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:750: seq_num 255 ranges {7372800=679936,8052736=548864,8601600=598016,9199616=57344} 2024-09-13T14:21:28.850 INFO:tasks.rados.rados.0.smithi026.stdout:750: writing smithi02683976-19 from 7372800 to 8052736 tid 1 2024-09-13T14:21:28.853 INFO:tasks.rados.rados.0.smithi026.stdout:750: writing smithi02683976-19 from 8052736 to 8601600 tid 2 2024-09-13T14:21:28.857 INFO:tasks.rados.rados.0.smithi026.stdout:750: writing smithi02683976-19 from 8601600 to 9199616 tid 3 2024-09-13T14:21:28.857 INFO:tasks.rados.rados.0.smithi026.stdout:750: writing smithi02683976-19 from 9199616 to 9256960 tid 4 2024-09-13T14:21:28.857 INFO:tasks.rados.rados.0.smithi026.stdout:751: rmattr oid 30 current snap is 66 2024-09-13T14:21:28.857 INFO:tasks.rados.rados.0.smithi026.stdout:751: done (3 left) 2024-09-13T14:21:28.857 INFO:tasks.rados.rados.0.smithi026.stdout:752: snap_remove snap 64 2024-09-13T14:21:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[27770]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:21:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[27770]: pgmap v66: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[27770]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:21:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[39182]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:21:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[39182]: pgmap v66: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:29.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:28 smithi026 ceph-mon[39182]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:21:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:28 smithi145 ceph-mon[31057]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:21:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:28 smithi145 ceph-mon[31057]: pgmap v66: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:28 smithi145 ceph-mon[31057]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:21:29.803 INFO:tasks.rados.rados.0.smithi026.stdout:752: done (3 left) 2024-09-13T14:21:29.803 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 66 2024-09-13T14:21:29.803 INFO:tasks.rados.rados.0.smithi026.stdout:753: seq_num 256 ranges {0=696320,696320=434176,1130496=770048} 2024-09-13T14:21:29.807 INFO:tasks.rados.rados.0.smithi026.stdout:753: writing smithi02683976-7 from 0 to 696320 tid 1 2024-09-13T14:21:29.810 INFO:tasks.rados.rados.0.smithi026.stdout:753: writing smithi02683976-7 from 696320 to 1130496 tid 2 2024-09-13T14:21:29.814 INFO:tasks.rados.rados.0.smithi026.stdout:753: writing smithi02683976-7 from 1130496 to 1900544 tid 3 2024-09-13T14:21:29.815 INFO:tasks.rados.rados.0.smithi026.stdout:754: rollback oid 47 current snap is 66 2024-09-13T14:21:29.815 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 65 2024-09-13T14:21:29.815 INFO:tasks.rados.rados.0.smithi026.stdout:749: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:21:29.857 INFO:tasks.rados.rados.0.smithi026.stdout:748: expect (ObjNum 77 snap 10 seq_num 77) 2024-09-13T14:21:29.924 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 1 to smithi02683976-19 2024-09-13T14:21:29.924 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 2 to smithi02683976-19 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 3 to smithi02683976-19 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 4 to smithi02683976-19 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:748: done (4 left) 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:749: done (3 left) 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:755: snap_create 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 5 to smithi02683976-19 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:750: finishing write tid 6 to smithi02683976-19 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 415 (ObjNum 255 snap 66 seq_num 255) dirty exists 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:750: left oid 19 (ObjNum 255 snap 66 seq_num 255) 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:753: finishing write tid 1 to smithi02683976-7 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:753: finishing write tid 2 to smithi02683976-7 2024-09-13T14:21:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:753: finishing write tid 3 to smithi02683976-7 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:753: finishing write tid 4 to smithi02683976-7 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:753: finishing write tid 5 to smithi02683976-7 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 36 (ObjNum 256 snap 66 seq_num 256) dirty exists 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:753: left oid 7 (ObjNum 256 snap 66 seq_num 256) 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:754: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:754: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:754: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:21:29.926 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 110 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-13T14:21:30.809 INFO:tasks.rados.rados.0.smithi026.stdout:750: done (3 left) 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:753: done (2 left) 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:754: done (1 left) 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:755: done (0 left) 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:756: copy_from oid 48 from oid 7 current snap is 67 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:757: read oid 2 snap -1 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:757: expect deleted 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 67 2024-09-13T14:21:30.810 INFO:tasks.rados.rados.0.smithi026.stdout:758: seq_num 257 ranges {1761280=712704,2473984=720896,3194880=483328} 2024-09-13T14:21:30.824 INFO:tasks.rados.rados.0.smithi026.stdout:758: writing smithi02683976-13 from 1761280 to 2473984 tid 1 2024-09-13T14:21:30.828 INFO:tasks.rados.rados.0.smithi026.stdout:758: writing smithi02683976-13 from 2473984 to 3194880 tid 2 2024-09-13T14:21:30.831 INFO:tasks.rados.rados.0.smithi026.stdout:758: writing smithi02683976-13 from 3194880 to 3678208 tid 3 2024-09-13T14:21:30.831 INFO:tasks.rados.rados.0.smithi026.stdout:759: read oid 47 snap -1 2024-09-13T14:21:30.831 INFO:tasks.rados.rados.0.smithi026.stdout:759: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:760: setattr oid 20 current snap is 67 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:756: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:756: finishing copy_from to smithi02683976-48 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 36 (ObjNum 256 snap 66 seq_num 256) dirty exists 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:758: finishing write tid 1 to smithi02683976-13 2024-09-13T14:21:30.832 INFO:tasks.rados.rados.0.smithi026.stdout:758: finishing write tid 2 to smithi02683976-13 2024-09-13T14:21:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:756: done (4 left) 2024-09-13T14:21:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:757: done (3 left) 2024-09-13T14:21:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:761: read oid 19 snap 61 2024-09-13T14:21:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:761: expect (ObjNum 235 snap 58 seq_num 235) 2024-09-13T14:21:30.833 INFO:tasks.rados.rados.0.smithi026.stdout:762: delete oid 23 current snap is 67 2024-09-13T14:21:30.836 INFO:tasks.rados.rados.0.smithi026.stdout:762: done (4 left) 2024-09-13T14:21:30.836 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 67 2024-09-13T14:21:30.836 INFO:tasks.rados.rados.0.smithi026.stdout:763: seq_num 259 ranges {6242304=745472,6987776=540672,7528448=581632,8110080=81920} 2024-09-13T14:21:30.877 INFO:tasks.rados.rados.0.smithi026.stdout:763: writing smithi02683976-31 from 6242304 to 6987776 tid 1 2024-09-13T14:21:30.878 INFO:tasks.rados.rados.0.smithi026.stdout:763: writing smithi02683976-31 from 6987776 to 7528448 tid 2 2024-09-13T14:21:30.882 INFO:tasks.rados.rados.0.smithi026.stdout:763: writing smithi02683976-31 from 7528448 to 8110080 tid 3 2024-09-13T14:21:30.882 INFO:tasks.rados.rados.0.smithi026.stdout:763: writing smithi02683976-31 from 8110080 to 8192000 tid 4 2024-09-13T14:21:30.883 INFO:tasks.rados.rados.0.smithi026.stdout:764: copy_from oid 37 from oid 45 current snap is 67 2024-09-13T14:21:30.883 INFO:tasks.rados.rados.0.smithi026.stdout:759: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:758: finishing write tid 3 to smithi02683976-13 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:758: finishing write tid 4 to smithi02683976-13 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 424 (ObjNum 33 snap 0 seq_num 1126792472) dirty exists 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:758: finishing write tid 5 to smithi02683976-13 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 422 (ObjNum 257 snap 67 seq_num 257) dirty exists 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:758: left oid 13 (ObjNum 257 snap 67 seq_num 257) 2024-09-13T14:21:30.929 INFO:tasks.rados.rados.0.smithi026.stdout:761: expect (ObjNum 235 snap 58 seq_num 235) 2024-09-13T14:21:31.066 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:30 smithi145 ceph-mon[31057]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:21:31.066 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:30 smithi145 ceph-mon[31057]: pgmap v69: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:31.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:30 smithi026 ceph-mon[27770]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:21:31.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:30 smithi026 ceph-mon[27770]: pgmap v69: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:30 smithi026 ceph-mon[39182]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:21:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:30 smithi026 ceph-mon[39182]: pgmap v69: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 MiB/s wr, 23 op/s 2024-09-13T14:21:31.387 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 1 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 2 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 3 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 4 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 5 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: finishing write tid 6 to smithi02683976-31 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 410 (ObjNum 259 snap 67 seq_num 259) dirty exists 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:763: left oid 31 (ObjNum 259 snap 67 seq_num 259) 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:764: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:21:31.388 INFO:tasks.rados.rados.0.smithi026.stdout:764: finishing copy_from to smithi02683976-37 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 278 (ObjNum 200 snap 48 seq_num 200) dirty exists 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:758: done (5 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:759: done (4 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:760: done (3 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:761: done (2 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:763: done (1 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:764: done (0 left) 2024-09-13T14:21:31.389 INFO:tasks.rados.rados.0.smithi026.stdout:765: snap_remove snap 65 2024-09-13T14:21:31.814 INFO:tasks.rados.rados.0.smithi026.stdout:765: done (0 left) 2024-09-13T14:21:31.814 INFO:tasks.rados.rados.0.smithi026.stdout:766: read oid 7 snap -1 2024-09-13T14:21:31.814 INFO:tasks.rados.rados.0.smithi026.stdout:766: expect (ObjNum 256 snap 66 seq_num 256) 2024-09-13T14:21:31.814 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 67 2024-09-13T14:21:31.814 INFO:tasks.rados.rados.0.smithi026.stdout:767: seq_num 260 ranges {0=761856,761856=417792,1179648=786432} 2024-09-13T14:21:31.819 INFO:tasks.rados.rados.0.smithi026.stdout:767: writing smithi02683976-34 from 0 to 761856 tid 1 2024-09-13T14:21:31.822 INFO:tasks.rados.rados.0.smithi026.stdout:767: writing smithi02683976-34 from 761856 to 1179648 tid 2 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:767: writing smithi02683976-34 from 1179648 to 1966080 tid 3 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:768: read oid 11 snap 66 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:768: expect (ObjNum 252 snap 66 seq_num 252) 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:769: rollback oid 25 current snap is 67 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 63 2024-09-13T14:21:31.828 INFO:tasks.rados.rados.0.smithi026.stdout:766: expect (ObjNum 256 snap 66 seq_num 256) 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:767: finishing write tid 1 to smithi02683976-34 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:767: finishing write tid 2 to smithi02683976-34 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:766: done (3 left) 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:770: read oid 37 snap -1 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:770: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:771: rollback oid 39 current snap is 67 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 62 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:769: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:769: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:769: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:21:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 206 (ObjNum 218 snap 53 seq_num 218) dirty exists 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:767: finishing write tid 3 to smithi02683976-34 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:769: done (4 left) 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:772: rollback oid 46 current snap is 67 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 66 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:767: finishing write tid 4 to smithi02683976-34 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:767: finishing write tid 5 to smithi02683976-34 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 247 (ObjNum 260 snap 67 seq_num 260) dirty exists 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:767: left oid 34 (ObjNum 260 snap 67 seq_num 260) 2024-09-13T14:21:31.885 INFO:tasks.rados.rados.0.smithi026.stdout:768: expect (ObjNum 252 snap 66 seq_num 252) 2024-09-13T14:21:31.981 INFO:tasks.rados.rados.0.smithi026.stdout:771: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:772: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:771: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:772: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:771: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 210 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:772: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 245 (ObjNum 233 snap 58 seq_num 233) dirty exists 2024-09-13T14:21:31.982 INFO:tasks.rados.rados.0.smithi026.stdout:770: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:31 smithi026 ceph-mon[39182]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:21:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:31 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:32.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:31 smithi026 ceph-mon[27770]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:21:32.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:31 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:32.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:31 smithi145 ceph-mon[31057]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:21:32.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:31 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:32.289 INFO:tasks.rados.rados.0.smithi026.stdout:767: done (4 left) 2024-09-13T14:21:32.289 INFO:tasks.rados.rados.0.smithi026.stdout:768: done (3 left) 2024-09-13T14:21:32.290 INFO:tasks.rados.rados.0.smithi026.stdout:770: done (2 left) 2024-09-13T14:21:32.290 INFO:tasks.rados.rados.0.smithi026.stdout:771: done (1 left) 2024-09-13T14:21:32.290 INFO:tasks.rados.rados.0.smithi026.stdout:772: done (0 left) 2024-09-13T14:21:32.290 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 67 2024-09-13T14:21:32.290 INFO:tasks.rados.rados.0.smithi026.stdout:773: seq_num 261 ranges {0=778240,778240=704512,1482752=499712} 2024-09-13T14:21:32.293 INFO:tasks.rados.rados.0.smithi026.stdout:773: writing smithi02683976-17 from 0 to 778240 tid 1 2024-09-13T14:21:32.297 INFO:tasks.rados.rados.0.smithi026.stdout:773: writing smithi02683976-17 from 778240 to 1482752 tid 2 2024-09-13T14:21:32.300 INFO:tasks.rados.rados.0.smithi026.stdout:773: writing smithi02683976-17 from 1482752 to 1982464 tid 3 2024-09-13T14:21:32.300 INFO:tasks.rados.rados.0.smithi026.stdout:774: copy_from oid 46 from oid 19 current snap is 67 2024-09-13T14:21:32.300 INFO:tasks.rados.rados.0.smithi026.stdout:773: finishing write tid 1 to smithi02683976-17 2024-09-13T14:21:32.301 INFO:tasks.rados.rados.0.smithi026.stdout:775: snap_remove snap 66 2024-09-13T14:21:32.818 INFO:tasks.rados.rados.0.smithi026.stdout:775: done (2 left) 2024-09-13T14:21:32.818 INFO:tasks.rados.rados.0.smithi026.stdout:776: copy_from oid 24 from oid 22 current snap is 67 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: finishing write tid 2 to smithi02683976-17 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: finishing write tid 3 to smithi02683976-17 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: finishing write tid 4 to smithi02683976-17 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: finishing write tid 5 to smithi02683976-17 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 251 (ObjNum 261 snap 67 seq_num 261) dirty exists 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: left oid 17 (ObjNum 261 snap 67 seq_num 261) 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:774: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:774: finishing copy_from to smithi02683976-46 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 415 (ObjNum 255 snap 66 seq_num 255) dirty exists 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:773: done (2 left) 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:774: done (1 left) 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:777: copy_from oid 43 from oid 35 current snap is 67 2024-09-13T14:21:32.819 INFO:tasks.rados.rados.0.smithi026.stdout:778: setattr oid 29 current snap is 67 2024-09-13T14:21:32.820 INFO:tasks.rados.rados.0.smithi026.stdout:779: setattr oid 32 current snap is 67 2024-09-13T14:21:32.823 INFO:tasks.rados.rados.0.smithi026.stdout:780: snap_create 2024-09-13T14:21:32.825 INFO:tasks.rados.rados.0.smithi026.stdout:777: finishing copy_from to smithi02683976-43 2024-09-13T14:21:32.825 INFO:tasks.rados.rados.0.smithi026.stdout:777: got expected ENOENT (src dne) 2024-09-13T14:21:32.826 INFO:tasks.rados.rados.0.smithi026.stdout:777: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:21:32.827 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 436 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:21:32.828 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 377 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:32.833 INFO:tasks.rados.rados.0.smithi026.stdout:776: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:21:32.834 INFO:tasks.rados.rados.0.smithi026.stdout:776: finishing copy_from to smithi02683976-24 2024-09-13T14:21:32.834 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 76 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:21:32.925 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 systemd[1]: Stopping Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:32.926 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-mon[31057]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:21:32.926 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-mon[31057]: pgmap v72: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:32.926 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:32] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-09-13T14:21:33.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:32 smithi026 ceph-mon[27770]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:21:33.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:32 smithi026 ceph-mon[27770]: pgmap v72: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:32 smithi026 ceph-mon[39182]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:21:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:32 smithi026 ceph-mon[39182]: pgmap v72: 121 pgs: 121 active+clean; 100 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:33.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:21:33.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[64453]: ts=2024-09-13T14:21:32.925Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:21:33.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:32 smithi145 podman[76079]: 2024-09-13 14:21:32.935735756 +0000 UTC m=+0.148641367 container died 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:33.591 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 podman[76079]: 2024-09-13 14:21:33.339591897 +0000 UTC m=+0.552497549 container cleanup 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:33.591 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 bash[76079]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:21:33.820 INFO:tasks.rados.rados.0.smithi026.stdout:776: done (4 left) 2024-09-13T14:21:33.820 INFO:tasks.rados.rados.0.smithi026.stdout:777: done (3 left) 2024-09-13T14:21:33.820 INFO:tasks.rados.rados.0.smithi026.stdout:778: done (2 left) 2024-09-13T14:21:33.820 INFO:tasks.rados.rados.0.smithi026.stdout:779: done (1 left) 2024-09-13T14:21:33.821 INFO:tasks.rados.rados.0.smithi026.stdout:780: done (0 left) 2024-09-13T14:21:33.821 INFO:tasks.rados.rados.0.smithi026.stdout:781: rollback oid 27 current snap is 68 2024-09-13T14:21:33.821 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 62 2024-09-13T14:21:33.821 INFO:tasks.rados.rados.0.smithi026.stdout:782: snap_remove snap 67 2024-09-13T14:21:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:33 smithi145 ceph-mon[31057]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:21:33.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 podman[76096]: 2024-09-13 14:21:33.59081134 +0000 UTC m=+0.656218916 container remove 33771513b4fc5d4669a40eba31acecd7509e9cfca238f7c46660b5c69cd17eda (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:33.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:21:33.956 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 systemd[1]: Stopped Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:33.956 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.760s CPU time. 2024-09-13T14:21:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:33 smithi026 ceph-mon[27770]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:21:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:33 smithi026 ceph-mon[39182]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:21:34.287 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:33 smithi145 systemd[1]: Starting Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:34.287 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 podman[76231]: 2024-09-13 14:21:34.084250934 +0000 UTC m=+0.016611234 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:21:34.287 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 podman[76231]: 2024-09-13 14:21:34.286696212 +0000 UTC m=+0.219056541 container create 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:782: done (1 left) 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:783: copy_from oid 11 from oid 41 current snap is 68 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:781: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:781: done (1 left) 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:784: copy_from oid 46 from oid 17 current snap is 68 2024-09-13T14:21:34.832 INFO:tasks.rados.rados.0.smithi026.stdout:785: delete oid 29 current snap is 68 2024-09-13T14:21:34.837 INFO:tasks.rados.rados.0.smithi026.stdout:785: done (2 left) 2024-09-13T14:21:34.837 INFO:tasks.rados.rados.0.smithi026.stdout:786: snap_remove snap 63 2024-09-13T14:21:35.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:34 smithi026 ceph-mon[27770]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:21:35.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:34 smithi026 ceph-mon[27770]: pgmap v75: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:34 smithi026 ceph-mon[39182]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:21:35.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:34 smithi026 ceph-mon[39182]: pgmap v75: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:35.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-mon[31057]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:21:35.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-mon[31057]: pgmap v75: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:35.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 podman[76231]: 2024-09-13 14:21:34.723982626 +0000 UTC m=+0.656342914 container init 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:35.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 podman[76231]: 2024-09-13 14:21:34.729639643 +0000 UTC m=+0.661999931 container start 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:21:35.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.751Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:21:35.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.751Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:21:35.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.751Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi145 (none))" 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.751Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.751Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.753Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.753Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.756Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.756Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.757Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.757Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.89µs 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.757Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.757Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.770Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.771Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T14:21:35.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.771Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=43.079µs wal_replay_duration=13.724477ms wbl_replay_duration=131ns total_replay_duration=13.794759ms 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.774Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.774Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.774Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.790Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.478512ms db_storage=1.359µs remote_storage=1.635µs web_handler=510ns query_engine=830ns scrape=459.854µs scrape_sd=100.438µs notify=17.411µs notify_sd=345.465µs rules=15.175386ms tracing=5.828µs 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.790Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:21:35.208 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:34.790Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:786: done (2 left) 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 68 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:784: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:783: finishing copy_from racing read to smithi02683976-11 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:783: finishing copy_from to smithi02683976-11 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 137 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:784: finishing copy_from to smithi02683976-46 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 251 (ObjNum 261 snap 67 seq_num 261) dirty exists 2024-09-13T14:21:35.841 INFO:tasks.rados.rados.0.smithi026.stdout:787: seq_num 263 ranges {237568=409600,647168=475136,1122304=557056,1679360=573440} 2024-09-13T14:21:35.844 INFO:tasks.rados.rados.0.smithi026.stdout:787: writing smithi02683976-12 from 237568 to 647168 tid 1 2024-09-13T14:21:35.847 INFO:tasks.rados.rados.0.smithi026.stdout:787: writing smithi02683976-12 from 647168 to 1122304 tid 2 2024-09-13T14:21:35.850 INFO:tasks.rados.rados.0.smithi026.stdout:787: writing smithi02683976-12 from 1122304 to 1679360 tid 3 2024-09-13T14:21:35.854 INFO:tasks.rados.rados.0.smithi026.stdout:787: writing smithi02683976-12 from 1679360 to 2252800 tid 4 2024-09-13T14:21:35.854 INFO:tasks.rados.rados.0.smithi026.stdout:783: done (2 left) 2024-09-13T14:21:35.854 INFO:tasks.rados.rados.0.smithi026.stdout:784: done (1 left) 2024-09-13T14:21:35.854 INFO:tasks.rados.rados.0.smithi026.stdout:788: delete oid 3 current snap is 68 2024-09-13T14:21:35.854 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 1 to smithi02683976-12 2024-09-13T14:21:35.856 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 2 to smithi02683976-12 2024-09-13T14:21:35.857 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 3 to smithi02683976-12 2024-09-13T14:21:35.858 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 4 to smithi02683976-12 2024-09-13T14:21:35.859 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 5 to smithi02683976-12 2024-09-13T14:21:35.860 INFO:tasks.rados.rados.0.smithi026.stdout:788: done (1 left) 2024-09-13T14:21:35.860 INFO:tasks.rados.rados.0.smithi026.stdout:789: snap_remove snap 61 2024-09-13T14:21:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:35 smithi145 ceph-mon[31057]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:21:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:35 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:35 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:35.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:35 smithi145 bash[76231]: 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 2024-09-13T14:21:35.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:35 smithi145 systemd[1]: Started Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:36.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[27770]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:21:36.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:36.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:36.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[39182]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:21:36.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:36.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:35 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:789: done (1 left) 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:790: read oid 9 snap -1 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:790: expect (ObjNum 161 snap 38 seq_num 161) 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:787: finishing write tid 6 to smithi02683976-12 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 161 (ObjNum 263 snap 68 seq_num 263) dirty exists 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:787: left oid 12 (ObjNum 263 snap 68 seq_num 263) 2024-09-13T14:21:36.846 INFO:tasks.rados.rados.0.smithi026.stdout:787: done (1 left) 2024-09-13T14:21:36.847 INFO:tasks.rados.rados.0.smithi026.stdout:791: snap_remove snap 62 2024-09-13T14:21:36.948 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:36 smithi145 ceph-mon[31057]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:21:36.948 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:36 smithi145 ceph-mon[31057]: Reconfiguring daemon grafana.a on smithi145 2024-09-13T14:21:36.948 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:36 smithi145 ceph-mon[31057]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:21:36.948 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:36 smithi145 ceph-mon[31057]: pgmap v78: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[39182]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:21:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[39182]: Reconfiguring daemon grafana.a on smithi145 2024-09-13T14:21:37.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[39182]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:21:37.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[39182]: pgmap v78: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:37.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[27770]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:21:37.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[27770]: Reconfiguring daemon grafana.a on smithi145 2024-09-13T14:21:37.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[27770]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:21:37.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:36 smithi026 ceph-mon[27770]: pgmap v78: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-09-13T14:21:37.848 INFO:tasks.rados.rados.0.smithi026.stdout:790: expect (ObjNum 161 snap 38 seq_num 161) 2024-09-13T14:21:37.862 INFO:tasks.rados.rados.0.smithi026.stdout:790: done (1 left) 2024-09-13T14:21:37.862 INFO:tasks.rados.rados.0.smithi026.stdout:791: done (0 left) 2024-09-13T14:21:37.862 INFO:tasks.rados.rados.0.smithi026.stdout:792: read oid 35 snap -1 2024-09-13T14:21:37.863 INFO:tasks.rados.rados.0.smithi026.stdout:792: expect deleted 2024-09-13T14:21:37.863 INFO:tasks.rados.rados.0.smithi026.stdout:793: rollback oid 4 current snap is 68 2024-09-13T14:21:37.863 INFO:tasks.rados.rados.0.smithi026.stdout:793: done (1 left) 2024-09-13T14:21:37.863 INFO:tasks.rados.rados.0.smithi026.stdout:794: delete oid 17 current snap is 68 2024-09-13T14:21:37.866 INFO:tasks.rados.rados.0.smithi026.stdout:792: done (1 left) 2024-09-13T14:21:37.866 INFO:tasks.rados.rados.0.smithi026.stdout:794: done (0 left) 2024-09-13T14:21:37.866 INFO:tasks.rados.rados.0.smithi026.stdout:795: read oid 2 snap -1 2024-09-13T14:21:37.866 INFO:tasks.rados.rados.0.smithi026.stdout:795: expect deleted 2024-09-13T14:21:37.867 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 68 2024-09-13T14:21:37.867 INFO:tasks.rados.rados.0.smithi026.stdout:796: seq_num 266 ranges {2539520=458752,2998272=524288,3522560=729088,4251648=352256} 2024-09-13T14:21:37.882 INFO:tasks.rados.rados.0.smithi026.stdout:796: writing smithi02683976-26 from 2539520 to 2998272 tid 1 2024-09-13T14:21:37.885 INFO:tasks.rados.rados.0.smithi026.stdout:796: writing smithi02683976-26 from 2998272 to 3522560 tid 2 2024-09-13T14:21:37.889 INFO:tasks.rados.rados.0.smithi026.stdout:796: writing smithi02683976-26 from 3522560 to 4251648 tid 3 2024-09-13T14:21:37.891 INFO:tasks.rados.rados.0.smithi026.stdout:796: writing smithi02683976-26 from 4251648 to 4603904 tid 4 2024-09-13T14:21:37.891 INFO:tasks.rados.rados.0.smithi026.stdout:797: rollback oid 27 current snap is 68 2024-09-13T14:21:37.891 INFO:tasks.rados.rados.0.smithi026.stdout:797: done (2 left) 2024-09-13T14:21:37.891 INFO:tasks.rados.rados.0.smithi026.stdout:798: snap_create 2024-09-13T14:21:37.891 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 1 to smithi02683976-26 2024-09-13T14:21:37.892 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 2 to smithi02683976-26 2024-09-13T14:21:37.893 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 3 to smithi02683976-26 2024-09-13T14:21:37.893 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 4 to smithi02683976-26 2024-09-13T14:21:37.894 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 5 to smithi02683976-26 2024-09-13T14:21:37.895 INFO:tasks.rados.rados.0.smithi026.stdout:796: finishing write tid 6 to smithi02683976-26 2024-09-13T14:21:37.895 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 170 (ObjNum 266 snap 68 seq_num 266) dirty exists 2024-09-13T14:21:37.895 INFO:tasks.rados.rados.0.smithi026.stdout:796: left oid 26 (ObjNum 266 snap 68 seq_num 266) 2024-09-13T14:21:37.968 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:37 smithi145 ceph-mon[31057]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:21:38.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:37 smithi026 ceph-mon[27770]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:21:38.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:37 smithi026 ceph-mon[39182]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:21:38.753 INFO:tasks.rados.rados.0.smithi026.stdout:795: done (2 left) 2024-09-13T14:21:38.753 INFO:tasks.rados.rados.0.smithi026.stdout:796: done (1 left) 2024-09-13T14:21:38.753 INFO:tasks.rados.rados.0.smithi026.stdout:798: done (0 left) 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:799: read oid 5 snap -1 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:799: expect deleted 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:800: copy_from oid 36 from oid 43 current snap is 69 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:801: rmattr oid 27 current snap is 69 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:801: done (2 left) 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:802: rollback oid 8 current snap is 69 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 68 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 69 2024-09-13T14:21:38.754 INFO:tasks.rados.rados.0.smithi026.stdout:803: seq_num 267 ranges {0=475136,475136=458752,933888=540672,1474560=606208} 2024-09-13T14:21:38.756 INFO:tasks.rados.rados.0.smithi026.stdout:803: writing smithi02683976-10 from 0 to 475136 tid 1 2024-09-13T14:21:38.759 INFO:tasks.rados.rados.0.smithi026.stdout:803: writing smithi02683976-10 from 475136 to 933888 tid 2 2024-09-13T14:21:38.763 INFO:tasks.rados.rados.0.smithi026.stdout:803: writing smithi02683976-10 from 933888 to 1474560 tid 3 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:803: writing smithi02683976-10 from 1474560 to 2080768 tid 4 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:804: read oid 32 snap -1 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:804: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:805: copy_from oid 38 from oid 29 current snap is 69 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:802: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:799: done (5 left) 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:806: rollback oid 25 current snap is 69 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 68 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 1 to smithi02683976-10 2024-09-13T14:21:38.769 INFO:tasks.rados.rados.0.smithi026.stdout:807: read oid 12 snap -1 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:807: expect (ObjNum 263 snap 68 seq_num 263) 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:802: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:802: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 204 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 2 to smithi02683976-10 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 3 to smithi02683976-10 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:802: done (6 left) 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:808: read oid 34 snap -1 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:808: expect (ObjNum 260 snap 67 seq_num 260) 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:809: rollback oid 6 current snap is 69 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 68 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 69 2024-09-13T14:21:38.770 INFO:tasks.rados.rados.0.smithi026.stdout:810: seq_num 268 ranges {1867776=491520,2359296=745472,3104768=745472,3850240=114688} 2024-09-13T14:21:38.781 INFO:tasks.rados.rados.0.smithi026.stdout:810: writing smithi02683976-11 from 1867776 to 2359296 tid 1 2024-09-13T14:21:38.786 INFO:tasks.rados.rados.0.smithi026.stdout:810: writing smithi02683976-11 from 2359296 to 3104768 tid 2 2024-09-13T14:21:38.790 INFO:tasks.rados.rados.0.smithi026.stdout:810: writing smithi02683976-11 from 3104768 to 3850240 tid 3 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:810: writing smithi02683976-11 from 3850240 to 3964928 tid 4 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:811: delete oid 46 current snap is 69 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:806: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:800: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 4 to smithi02683976-10 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 5 to smithi02683976-10 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:806: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:806: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 238 (ObjNum 218 snap 53 seq_num 218) dirty exists 2024-09-13T14:21:38.792 INFO:tasks.rados.rados.0.smithi026.stdout:809: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:805: finishing copy_from to smithi02683976-38 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:805: got expected ENOENT (src dne) 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:805: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:809: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:809: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 441 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:803: finishing write tid 6 to smithi02683976-10 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 245 (ObjNum 267 snap 69 seq_num 267) dirty exists 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:803: left oid 10 (ObjNum 267 snap 69 seq_num 267) 2024-09-13T14:21:38.793 INFO:tasks.rados.rados.0.smithi026.stdout:808: expect (ObjNum 260 snap 67 seq_num 260) 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:800: finishing copy_from to smithi02683976-36 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 137 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 1 to smithi02683976-11 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 2 to smithi02683976-11 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:800: done (9 left) 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:803: done (8 left) 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:805: done (7 left) 2024-09-13T14:21:38.850 INFO:tasks.rados.rados.0.smithi026.stdout:806: done (6 left) 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:808: done (5 left) 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:809: done (4 left) 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:811: done (3 left) 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:812: delete oid 22 current snap is 69 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 3 to smithi02683976-11 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 4 to smithi02683976-11 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 5 to smithi02683976-11 2024-09-13T14:21:38.851 INFO:tasks.rados.rados.0.smithi026.stdout:807: expect (ObjNum 263 snap 68 seq_num 263) 2024-09-13T14:21:38.913 INFO:tasks.rados.rados.0.smithi026.stdout:810: finishing write tid 6 to smithi02683976-11 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 447 (ObjNum 268 snap 69 seq_num 268) dirty exists 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:810: left oid 11 (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:807: done (3 left) 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:810: done (2 left) 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:812: done (1 left) 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 69 2024-09-13T14:21:38.914 INFO:tasks.rados.rados.0.smithi026.stdout:804: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[39182]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:21:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[39182]: pgmap v81: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:39.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[39182]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:21:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[27770]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:21:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[27770]: pgmap v81: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:39.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:38 smithi026 ceph-mon[27770]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:21:39.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:38 smithi145 ceph-mon[31057]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:21:39.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:38 smithi145 ceph-mon[31057]: pgmap v81: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:39.181 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:38 smithi145 ceph-mon[31057]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:21:39.181 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:38 smithi145 systemd[1]: Stopping Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:39.455 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:39 smithi145 podman[76539]: 2024-09-13 14:21:39.180489664 +0000 UTC m=+0.112516466 container died b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, name=ubi8, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.expose-services=, io.openshift.tags=base rhel8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-13T14:21:39.503 INFO:tasks.rados.rados.0.smithi026.stdout:813: seq_num 271 ranges {0=540672,540672=802816,1343488=516096,1859584=286720} 2024-09-13T14:21:39.506 INFO:tasks.rados.rados.0.smithi026.stdout:813: writing smithi02683976-4 from 0 to 540672 tid 1 2024-09-13T14:21:39.510 INFO:tasks.rados.rados.0.smithi026.stdout:813: writing smithi02683976-4 from 540672 to 1343488 tid 2 2024-09-13T14:21:39.513 INFO:tasks.rados.rados.0.smithi026.stdout:813: writing smithi02683976-4 from 1343488 to 1859584 tid 3 2024-09-13T14:21:39.515 INFO:tasks.rados.rados.0.smithi026.stdout:813: writing smithi02683976-4 from 1859584 to 2146304 tid 4 2024-09-13T14:21:39.515 INFO:tasks.rados.rados.0.smithi026.stdout:804: done (1 left) 2024-09-13T14:21:39.515 INFO:tasks.rados.rados.0.smithi026.stdout:814: rmattr oid 33 current snap is 69 2024-09-13T14:21:39.515 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 1 to smithi02683976-4 2024-09-13T14:21:39.515 INFO:tasks.rados.rados.0.smithi026.stdout:815: snap_create 2024-09-13T14:21:39.517 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 448 (ObjNum 65 snap 0 seq_num 509658064) dirty exists 2024-09-13T14:21:39.520 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 2 to smithi02683976-4 2024-09-13T14:21:39.520 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 3 to smithi02683976-4 2024-09-13T14:21:39.520 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 4 to smithi02683976-4 2024-09-13T14:21:39.520 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 5 to smithi02683976-4 2024-09-13T14:21:39.522 INFO:tasks.rados.rados.0.smithi026.stdout:813: finishing write tid 6 to smithi02683976-4 2024-09-13T14:21:39.522 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 189 (ObjNum 271 snap 69 seq_num 271) dirty exists 2024-09-13T14:21:39.522 INFO:tasks.rados.rados.0.smithi026.stdout:813: left oid 4 (ObjNum 271 snap 69 seq_num 271) 2024-09-13T14:21:39.858 INFO:tasks.rados.rados.0.smithi026.stdout:813: done (2 left) 2024-09-13T14:21:39.858 INFO:tasks.rados.rados.0.smithi026.stdout:814: done (1 left) 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:815: done (0 left) 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:816: rmattr oid 35 current snap is 70 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:816: done (0 left) 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:817: read oid 27 snap -1 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:817: expect deleted 2024-09-13T14:21:39.859 INFO:tasks.rados.rados.0.smithi026.stdout:818: setattr oid 18 current snap is 70 2024-09-13T14:21:39.860 INFO:tasks.rados.rados.0.smithi026.stdout:819: rollback oid 25 current snap is 70 2024-09-13T14:21:39.860 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 68 2024-09-13T14:21:39.860 INFO:tasks.rados.rados.0.smithi026.stdout:820: snap_remove snap 69 2024-09-13T14:21:39.891 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:39 smithi145 podman[76539]: 2024-09-13 14:21:39.62701272 +0000 UTC m=+0.559039523 container cleanup b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, version=8.5, name=ubi8, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, distribution-scope=public) 2024-09-13T14:21:39.891 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:39 smithi145 bash[76539]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:21:40.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:39 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:39] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-13T14:21:40.205 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:39 smithi145 podman[76555]: 2024-09-13 14:21:39.912555801 +0000 UTC m=+0.734300363 container remove b034aa05655942b9eb4c8d86cdd0b07738db7e051864cf215bdf3e10183fc361 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.component=ubi8-container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, version=8.5, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container) 2024-09-13T14:21:40.205 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:39 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:21:40.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:21:40.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 systemd[1]: Stopped Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:40.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Consumed 5.436s CPU time. 2024-09-13T14:21:40.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 systemd[1]: Starting Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:40.550 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 podman[76654]: 2024-09-13 14:21:40.153644059 +0000 UTC m=+0.015703101 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:21:40.550 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 podman[76654]: 2024-09-13 14:21:40.297449244 +0000 UTC m=+0.159508280 container create 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-type=git, version=8.5, architecture=x86_64, name=ubi8, distribution-scope=public, io.buildah.version=1.32.0, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner ) 2024-09-13T14:21:40.550 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 podman[76654]: 2024-09-13 14:21:40.549984131 +0000 UTC m=+0.412043173 container init 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.tags=base rhel8, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, maintainer=Paul Cuzner , distribution-scope=public, io.openshift.expose-services=) 2024-09-13T14:21:40.815 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 podman[76654]: 2024-09-13 14:21:40.555037071 +0000 UTC m=+0.417096107 container start 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, io.openshift.expose-services=, io.buildah.version=1.32.0, maintainer=Paul Cuzner , release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-13T14:21:40.815 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 bash[76654]: 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a 2024-09-13T14:21:40.815 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 systemd[1]: Started Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709461666Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.70964459Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709663563Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709671005Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709677002Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709683044Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.70968864Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709694561Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709700332Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:21:40.816 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709706026Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709711509Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709717575Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709723254Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=settings t=2024-09-13T14:21:40.709728849Z level=info msg="App mode production" 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=sqlstore t=2024-09-13T14:21:40.709778642Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=sqlstore t=2024-09-13T14:21:40.70979524Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=migrator t=2024-09-13T14:21:40.724101914Z level=info msg="Starting DB migrations" 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=migrator t=2024-09-13T14:21:40.728914262Z level=info msg="migrations completed" performed=0 skipped=481 duration=461.51µs 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=sqlstore t=2024-09-13T14:21:40.729372352Z level=info msg="Created default organization" 2024-09-13T14:21:40.817 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=plugin.loader t=2024-09-13T14:21:40.78682611Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:21:40.818 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=plugin.loader t=2024-09-13T14:21:40.814664508Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:21:40.818 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=plugin.loader t=2024-09-13T14:21:40.814695549Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:21:40.818 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=secrets t=2024-09-13T14:21:40.814732826Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:820: done (3 left) 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:821: copy_from oid 34 from oid 32 current snap is 70 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 378 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:819: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:819: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:819: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:21:40.866 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 242 (ObjNum 218 snap 53 seq_num 218) dirty exists 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:817: done (3 left) 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:818: done (2 left) 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:819: done (1 left) 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:822: read oid 27 snap -1 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:822: expect deleted 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 70 2024-09-13T14:21:40.867 INFO:tasks.rados.rados.0.smithi026.stdout:823: seq_num 272 ranges {0=565248,565248=688128,1253376=729088,1982464=188416} 2024-09-13T14:21:40.869 INFO:tasks.rados.rados.0.smithi026.stdout:823: writing smithi02683976-23 from 0 to 565248 tid 1 2024-09-13T14:21:40.873 INFO:tasks.rados.rados.0.smithi026.stdout:823: writing smithi02683976-23 from 565248 to 1253376 tid 2 2024-09-13T14:21:40.877 INFO:tasks.rados.rados.0.smithi026.stdout:823: writing smithi02683976-23 from 1253376 to 1982464 tid 3 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:823: writing smithi02683976-23 from 1982464 to 2170880 tid 4 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:824: copy_from oid 31 from oid 7 current snap is 70 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 1 to smithi02683976-23 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 2 to smithi02683976-23 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:822: done (3 left) 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:825: read oid 47 snap -1 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:825: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:21:40.880 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 70 2024-09-13T14:21:40.881 INFO:tasks.rados.rados.0.smithi026.stdout:826: seq_num 273 ranges {1851392=581632,2433024=565248,2998272=532480,3530752=507904} 2024-09-13T14:21:40.893 INFO:tasks.rados.rados.0.smithi026.stdout:826: writing smithi02683976-49 from 1851392 to 2433024 tid 1 2024-09-13T14:21:40.896 INFO:tasks.rados.rados.0.smithi026.stdout:826: writing smithi02683976-49 from 2433024 to 2998272 tid 2 2024-09-13T14:21:40.899 INFO:tasks.rados.rados.0.smithi026.stdout:826: writing smithi02683976-49 from 2998272 to 3530752 tid 3 2024-09-13T14:21:40.902 INFO:tasks.rados.rados.0.smithi026.stdout:826: writing smithi02683976-49 from 3530752 to 4038656 tid 4 2024-09-13T14:21:40.903 INFO:tasks.rados.rados.0.smithi026.stdout:827: read oid 20 snap 68 2024-09-13T14:21:40.903 INFO:tasks.rados.rados.0.smithi026.stdout:827: expect (ObjNum 519314560 snap 21981 seq_num 519057840) 2024-09-13T14:21:40.903 INFO:tasks.rados.rados.0.smithi026.stdout:828: rollback oid 19 current snap is 70 2024-09-13T14:21:40.903 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 68 2024-09-13T14:21:40.903 INFO:tasks.rados.rados.0.smithi026.stdout:825: expect (ObjNum 185 snap 44 seq_num 185) 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 3 to smithi02683976-23 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 4 to smithi02683976-23 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 5 to smithi02683976-23 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:823: finishing write tid 6 to smithi02683976-23 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 383 (ObjNum 272 snap 70 seq_num 272) dirty exists 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:823: left oid 23 (ObjNum 272 snap 70 seq_num 272) 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 1 to smithi02683976-49 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:824: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 2 to smithi02683976-49 2024-09-13T14:21:40.951 INFO:tasks.rados.rados.0.smithi026.stdout:824: finishing copy_from to smithi02683976-31 2024-09-13T14:21:40.952 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 36 (ObjNum 256 snap 66 seq_num 256) dirty exists 2024-09-13T14:21:40.952 INFO:tasks.rados.rados.0.smithi026.stdout:827: expect (ObjNum 519314560 snap 21981 seq_num 519057840) 2024-09-13T14:21:40.956 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 3 to smithi02683976-49 2024-09-13T14:21:40.956 INFO:tasks.rados.rados.0.smithi026.stdout:828: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:823: done (6 left) 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:824: done (5 left) 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:825: done (4 left) 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:827: done (3 left) 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 70 2024-09-13T14:21:40.957 INFO:tasks.rados.rados.0.smithi026.stdout:828: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:21:40.958 INFO:tasks.rados.rados.0.smithi026.stdout:829: seq_num 274 ranges {4603904=598016,5201920=499712,5701632=745472,6447104=360448} 2024-09-13T14:21:40.984 INFO:tasks.rados.rados.0.smithi026.stdout:829: writing smithi02683976-26 from 4603904 to 5201920 tid 1 2024-09-13T14:21:40.987 INFO:tasks.rados.rados.0.smithi026.stdout:829: writing smithi02683976-26 from 5201920 to 5701632 tid 2 2024-09-13T14:21:40.991 INFO:tasks.rados.rados.0.smithi026.stdout:829: writing smithi02683976-26 from 5701632 to 6447104 tid 3 2024-09-13T14:21:40.993 INFO:tasks.rados.rados.0.smithi026.stdout:829: writing smithi02683976-26 from 6447104 to 6807552 tid 4 2024-09-13T14:21:40.993 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 70 2024-09-13T14:21:40.993 INFO:tasks.rados.rados.0.smithi026.stdout:830: seq_num 275 ranges {0=614400,614400=786432,1400832=499712,1900544=319488} 2024-09-13T14:21:40.996 INFO:tasks.rados.rados.0.smithi026.stdout:830: writing smithi02683976-35 from 0 to 614400 tid 1 2024-09-13T14:21:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:830: writing smithi02683976-35 from 614400 to 1400832 tid 2 2024-09-13T14:21:41.003 INFO:tasks.rados.rados.0.smithi026.stdout:830: writing smithi02683976-35 from 1400832 to 1900544 tid 3 2024-09-13T14:21:41.006 INFO:tasks.rados.rados.0.smithi026.stdout:830: writing smithi02683976-35 from 1900544 to 2220032 tid 4 2024-09-13T14:21:41.006 INFO:tasks.rados.rados.0.smithi026.stdout:831: snap_create 2024-09-13T14:21:41.006 INFO:tasks.rados.rados.0.smithi026.stdout:828: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:21:41.006 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 469 (ObjNum 255 snap 66 seq_num 255) dirty exists 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 4 to smithi02683976-49 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:821: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 5 to smithi02683976-49 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:821: finishing copy_from to smithi02683976-34 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 377 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:826: finishing write tid 6 to smithi02683976-49 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 251 (ObjNum 273 snap 70 seq_num 273) dirty exists 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:826: left oid 49 (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 1 to smithi02683976-26 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 2 to smithi02683976-26 2024-09-13T14:21:41.007 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 3 to smithi02683976-26 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 4 to smithi02683976-26 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 5 to smithi02683976-26 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:829: finishing write tid 6 to smithi02683976-26 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 178 (ObjNum 274 snap 70 seq_num 274) dirty exists 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:829: left oid 26 (ObjNum 274 snap 70 seq_num 274) 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 1 to smithi02683976-35 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 2 to smithi02683976-35 2024-09-13T14:21:41.008 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 3 to smithi02683976-35 2024-09-13T14:21:41.009 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 4 to smithi02683976-35 2024-09-13T14:21:41.009 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 5 to smithi02683976-35 2024-09-13T14:21:41.009 INFO:tasks.rados.rados.0.smithi026.stdout:830: finishing write tid 6 to smithi02683976-35 2024-09-13T14:21:41.009 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 453 (ObjNum 275 snap 70 seq_num 275) dirty exists 2024-09-13T14:21:41.009 INFO:tasks.rados.rados.0.smithi026.stdout:830: left oid 35 (ObjNum 275 snap 70 seq_num 275) 2024-09-13T14:21:41.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: pgmap v84: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[39182]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: pgmap v84: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:21:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:40 smithi026 ceph-mon[27770]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:21:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:21:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: pgmap v84: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 223 KiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-13T14:21:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:21:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-mon[31057]: Deploying daemon node-exporter.b on smithi145 2024-09-13T14:21:41.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=query_data t=2024-09-13T14:21:40.816954693Z level=info msg="Query Service initialization" 2024-09-13T14:21:41.206 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=live.push_http t=2024-09-13T14:21:40.819361771Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:21:41.544 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:41 smithi145 systemd[1]: Stopping Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:41.831 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:41 smithi145 podman[76846]: 2024-09-13 14:21:41.543999806 +0000 UTC m=+0.132651558 container died 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=infra.usagestats.collector t=2024-09-13T14:21:41.593258474Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=provisioning.datasources t=2024-09-13T14:21:41.67899188Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=provisioning.datasources t=2024-09-13T14:21:41.679019805Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=provisioning.datasources t=2024-09-13T14:21:41.68920467Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=provisioning.alerting t=2024-09-13T14:21:41.770360485Z level=info msg="starting to provision alerting" 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=provisioning.alerting t=2024-09-13T14:21:41.770388756Z level=info msg="finished to provision alerting" 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=grafanaStorageLogger t=2024-09-13T14:21:41.770600439Z level=info msg="storage starting" 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=ngalert.state.manager t=2024-09-13T14:21:41.771092402Z level=info msg="Warming state cache for startup" 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=http.server t=2024-09-13T14:21:41.776666287Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=ngalert.state.manager t=2024-09-13T14:21:41.781965976Z level=info msg="State cache has been initialized" states=0 duration=10.873513ms 2024-09-13T14:21:41.832 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:21:41.782005076Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:21:41.833 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[76668]: logger=ticker t=2024-09-13T14:21:41.782029613Z level=info msg=starting first_tick=2024-09-13T14:21:50Z 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:821: done (5 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:826: done (4 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:828: done (3 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:829: done (2 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:830: done (1 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:831: done (0 left) 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 71 2024-09-13T14:21:41.870 INFO:tasks.rados.rados.0.smithi026.stdout:832: seq_num 276 ranges {5423104=630784,6053888=671744,6725632=704512,7430144=229376} 2024-09-13T14:21:41.902 INFO:tasks.rados.rados.0.smithi026.stdout:832: writing smithi02683976-45 from 5423104 to 6053888 tid 1 2024-09-13T14:21:41.906 INFO:tasks.rados.rados.0.smithi026.stdout:832: writing smithi02683976-45 from 6053888 to 6725632 tid 2 2024-09-13T14:21:41.910 INFO:tasks.rados.rados.0.smithi026.stdout:832: writing smithi02683976-45 from 6725632 to 7430144 tid 3 2024-09-13T14:21:41.911 INFO:tasks.rados.rados.0.smithi026.stdout:832: writing smithi02683976-45 from 7430144 to 7659520 tid 4 2024-09-13T14:21:41.912 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 71 2024-09-13T14:21:41.912 INFO:tasks.rados.rados.0.smithi026.stdout:833: seq_num 277 ranges {2080768=647168,2727936=557056,3284992=516096,3801088=532480} 2024-09-13T14:21:41.926 INFO:tasks.rados.rados.0.smithi026.stdout:833: writing smithi02683976-10 from 2080768 to 2727936 tid 1 2024-09-13T14:21:41.930 INFO:tasks.rados.rados.0.smithi026.stdout:833: writing smithi02683976-10 from 2727936 to 3284992 tid 2 2024-09-13T14:21:41.933 INFO:tasks.rados.rados.0.smithi026.stdout:833: writing smithi02683976-10 from 3284992 to 3801088 tid 3 2024-09-13T14:21:41.936 INFO:tasks.rados.rados.0.smithi026.stdout:833: writing smithi02683976-10 from 3801088 to 4333568 tid 4 2024-09-13T14:21:41.936 INFO:tasks.rados.rados.0.smithi026.stdout:834: delete oid 7 current snap is 71 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 1 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 2 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 3 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 4 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 5 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: finishing write tid 6 to smithi02683976-45 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 283 (ObjNum 276 snap 71 seq_num 276) dirty exists 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:832: left oid 45 (ObjNum 276 snap 71 seq_num 276) 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 1 to smithi02683976-10 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 2 to smithi02683976-10 2024-09-13T14:21:41.937 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 3 to smithi02683976-10 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 4 to smithi02683976-10 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:832: done (2 left) 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:834: done (1 left) 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:835: rmattr oid 20 current snap is 71 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 71 2024-09-13T14:21:41.939 INFO:tasks.rados.rados.0.smithi026.stdout:836: seq_num 279 ranges {1171456=679936,1851392=720896,2572288=483328,3055616=401408} 2024-09-13T14:21:41.949 INFO:tasks.rados.rados.0.smithi026.stdout:836: writing smithi02683976-47 from 1171456 to 1851392 tid 1 2024-09-13T14:21:41.953 INFO:tasks.rados.rados.0.smithi026.stdout:836: writing smithi02683976-47 from 1851392 to 2572288 tid 2 2024-09-13T14:21:41.956 INFO:tasks.rados.rados.0.smithi026.stdout:836: writing smithi02683976-47 from 2572288 to 3055616 tid 3 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:836: writing smithi02683976-47 from 3055616 to 3457024 tid 4 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 71 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 5 to smithi02683976-10 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:833: finishing write tid 6 to smithi02683976-10 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 257 (ObjNum 277 snap 71 seq_num 277) dirty exists 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:833: left oid 10 (ObjNum 277 snap 71 seq_num 277) 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 471 (ObjNum 33 snap 0 seq_num 509663312) dirty exists 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 1 to smithi02683976-47 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 2 to smithi02683976-47 2024-09-13T14:21:41.959 INFO:tasks.rados.rados.0.smithi026.stdout:837: seq_num 280 ranges {1900544=696320,2596864=606208,3203072=688128,3891200=311296} 2024-09-13T14:21:41.973 INFO:tasks.rados.rados.0.smithi026.stdout:837: writing smithi02683976-48 from 1900544 to 2596864 tid 1 2024-09-13T14:21:41.976 INFO:tasks.rados.rados.0.smithi026.stdout:837: writing smithi02683976-48 from 2596864 to 3203072 tid 2 2024-09-13T14:21:41.980 INFO:tasks.rados.rados.0.smithi026.stdout:837: writing smithi02683976-48 from 3203072 to 3891200 tid 3 2024-09-13T14:21:41.983 INFO:tasks.rados.rados.0.smithi026.stdout:837: writing smithi02683976-48 from 3891200 to 4202496 tid 4 2024-09-13T14:21:41.983 INFO:tasks.rados.rados.0.smithi026.stdout:833: done (3 left) 2024-09-13T14:21:41.983 INFO:tasks.rados.rados.0.smithi026.stdout:835: done (2 left) 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:838: delete oid 20 current snap is 71 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 3 to smithi02683976-47 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 4 to smithi02683976-47 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 5 to smithi02683976-47 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:836: finishing write tid 6 to smithi02683976-47 2024-09-13T14:21:41.984 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 125 (ObjNum 279 snap 71 seq_num 279) dirty exists 2024-09-13T14:21:41.985 INFO:tasks.rados.rados.0.smithi026.stdout:836: left oid 47 (ObjNum 279 snap 71 seq_num 279) 2024-09-13T14:21:41.985 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 1 to smithi02683976-48 2024-09-13T14:21:41.985 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 2 to smithi02683976-48 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:836: done (2 left) 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:838: done (1 left) 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:839: snap_create 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 3 to smithi02683976-48 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 4 to smithi02683976-48 2024-09-13T14:21:41.986 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 5 to smithi02683976-48 2024-09-13T14:21:41.987 INFO:tasks.rados.rados.0.smithi026.stdout:837: finishing write tid 6 to smithi02683976-48 2024-09-13T14:21:41.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 131 (ObjNum 280 snap 71 seq_num 280) dirty exists 2024-09-13T14:21:41.988 INFO:tasks.rados.rados.0.smithi026.stdout:837: left oid 48 (ObjNum 280 snap 71 seq_num 280) 2024-09-13T14:21:42.123 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:41 smithi145 ceph-mon[31057]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:21:42.135 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:41 smithi145 podman[76846]: 2024-09-13 14:21:41.9382652 +0000 UTC m=+0.526916950 container cleanup 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:42.136 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:41 smithi145 bash[76846]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:21:42.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:41 smithi026 ceph-mon[27770]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:21:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:41 smithi026 ceph-mon[39182]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:21:42.376 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 podman[76865]: 2024-09-13 14:21:42.123423173 +0000 UTC m=+0.583811919 container remove 02ae27c8e5f531dc6a469e7fc9e4e7d1406f6e34aec1305c5dc2f9f83bea1f32 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:42.376 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:21:42.376 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:21:42.376 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 systemd[1]: Stopped Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:42.376 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.252s CPU time. 2024-09-13T14:21:42.705 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 systemd[1]: Starting Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:42.875 INFO:tasks.rados.rados.0.smithi026.stdout:837: done (1 left) 2024-09-13T14:21:42.875 INFO:tasks.rados.rados.0.smithi026.stdout:839: done (0 left) 2024-09-13T14:21:42.876 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 72 2024-09-13T14:21:42.876 INFO:tasks.rados.rados.0.smithi026.stdout:840: seq_num 282 ranges {3678208=729088,4407296=425984,4833280=704512,5537792=475136} 2024-09-13T14:21:42.899 INFO:tasks.rados.rados.0.smithi026.stdout:840: writing smithi02683976-13 from 3678208 to 4407296 tid 1 2024-09-13T14:21:42.901 INFO:tasks.rados.rados.0.smithi026.stdout:840: writing smithi02683976-13 from 4407296 to 4833280 tid 2 2024-09-13T14:21:42.906 INFO:tasks.rados.rados.0.smithi026.stdout:840: writing smithi02683976-13 from 4833280 to 5537792 tid 3 2024-09-13T14:21:42.909 INFO:tasks.rados.rados.0.smithi026.stdout:840: writing smithi02683976-13 from 5537792 to 6012928 tid 4 2024-09-13T14:21:42.910 INFO:tasks.rados.rados.0.smithi026.stdout:841: delete oid 50 current snap is 72 2024-09-13T14:21:42.910 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 1 to smithi02683976-13 2024-09-13T14:21:42.910 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 2 to smithi02683976-13 2024-09-13T14:21:42.910 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 3 to smithi02683976-13 2024-09-13T14:21:42.912 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 4 to smithi02683976-13 2024-09-13T14:21:42.912 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 5 to smithi02683976-13 2024-09-13T14:21:42.912 INFO:tasks.rados.rados.0.smithi026.stdout:841: done (1 left) 2024-09-13T14:21:42.912 INFO:tasks.rados.rados.0.smithi026.stdout:842: rollback oid 21 current snap is 72 2024-09-13T14:21:42.913 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 68 2024-09-13T14:21:42.913 INFO:tasks.rados.rados.0.smithi026.stdout:843: snap_remove snap 70 2024-09-13T14:21:43.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:42 smithi145 ceph-mon[31057]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:21:43.102 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:42 smithi145 ceph-mon[31057]: pgmap v87: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:43.102 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:42 smithi145 bash[76998]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:21:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:42 smithi026 ceph-mon[27770]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:21:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:42 smithi026 ceph-mon[27770]: pgmap v87: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:42 smithi026 ceph-mon[39182]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:21:43.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:42 smithi026 ceph-mon[39182]: pgmap v87: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:43.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:43 smithi145 bash[76998]: Getting image source signatures 2024-09-13T14:21:43.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:43 smithi145 bash[76998]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:21:43.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:43 smithi145 bash[76998]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:21:43.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:43 smithi145 bash[76998]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:21:43.881 INFO:tasks.rados.rados.0.smithi026.stdout:843: done (2 left) 2024-09-13T14:21:43.881 INFO:tasks.rados.rados.0.smithi026.stdout:844: snap_create 2024-09-13T14:21:43.881 INFO:tasks.rados.rados.0.smithi026.stdout:840: finishing write tid 6 to smithi02683976-13 2024-09-13T14:21:43.881 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 478 (ObjNum 282 snap 72 seq_num 282) dirty exists 2024-09-13T14:21:43.881 INFO:tasks.rados.rados.0.smithi026.stdout:840: left oid 13 (ObjNum 282 snap 72 seq_num 282) 2024-09-13T14:21:43.882 INFO:tasks.rados.rados.0.smithi026.stdout:842: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:21:43.882 INFO:tasks.rados.rados.0.smithi026.stdout:842: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:21:43.882 INFO:tasks.rados.rados.0.smithi026.stdout:842: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:21:43.882 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 454 (ObjNum 226 snap 55 seq_num 226) dirty exists 2024-09-13T14:21:44.147 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:43 smithi145 ceph-mon[31057]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:21:44.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:43 smithi026 ceph-mon[39182]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:21:44.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:43 smithi026 ceph-mon[27770]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:21:44.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:44.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:44.880 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:44 smithi145 bash[76998]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:21:44.881 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:44 smithi145 bash[76998]: Writing manifest to image destination 2024-09-13T14:21:44.884 INFO:tasks.rados.rados.0.smithi026.stdout:840: done (2 left) 2024-09-13T14:21:44.884 INFO:tasks.rados.rados.0.smithi026.stdout:842: done (1 left) 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:844: done (0 left) 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:845: read oid 46 snap -1 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:845: expect deleted 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:846: read oid 12 snap -1 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:846: expect (ObjNum 263 snap 68 seq_num 263) 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:847: read oid 29 snap -1 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:847: expect deleted 2024-09-13T14:21:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:848: snap_remove snap 71 2024-09-13T14:21:45.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:44 smithi026 ceph-mon[39182]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:21:45.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:44 smithi026 ceph-mon[39182]: pgmap v90: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:45.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:44 smithi026 ceph-mon[27770]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:21:45.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:44 smithi026 ceph-mon[27770]: pgmap v90: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:44 smithi145 ceph-mon[31057]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:21:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:44 smithi145 ceph-mon[31057]: pgmap v90: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:45.537 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 podman[76998]: 2024-09-13 14:21:45.186268914 +0000 UTC m=+2.530297777 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:21:45.537 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 podman[76998]: 2024-09-13 14:21:45.293115323 +0000 UTC m=+2.637144186 container create a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:45.885 INFO:tasks.rados.rados.0.smithi026.stdout:848: done (3 left) 2024-09-13T14:21:45.886 INFO:tasks.rados.rados.0.smithi026.stdout:849: setattr oid 42 current snap is 73 2024-09-13T14:21:45.886 INFO:tasks.rados.rados.0.smithi026.stdout:846: expect (ObjNum 263 snap 68 seq_num 263) 2024-09-13T14:21:45.892 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:45.892 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 podman[76998]: 2024-09-13 14:21:45.53679138 +0000 UTC m=+2.880820247 container init a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 podman[76998]: 2024-09-13 14:21:45.541934148 +0000 UTC m=+2.885963015 container start a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.545Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.545Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.545Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.545Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:21:45.893 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:21:45.894 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:21:45.895 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:21:45.896 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:21:45.897 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:21:45.898 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:21:45.899 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b[77070]: ts=2024-09-13T14:21:45.546Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:21:45.899 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 bash[76998]: a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d 2024-09-13T14:21:45.899 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:21:45 smithi145 systemd[1]: Started Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:45.951 INFO:tasks.rados.rados.0.smithi026.stdout:845: done (3 left) 2024-09-13T14:21:45.951 INFO:tasks.rados.rados.0.smithi026.stdout:846: done (2 left) 2024-09-13T14:21:45.951 INFO:tasks.rados.rados.0.smithi026.stdout:847: done (1 left) 2024-09-13T14:21:45.951 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 73 2024-09-13T14:21:45.951 INFO:tasks.rados.rados.0.smithi026.stdout:850: seq_num 284 ranges {2252800=761856,3014656=598016,3612672=671744,4284416=335872} 2024-09-13T14:21:45.969 INFO:tasks.rados.rados.0.smithi026.stdout:850: writing smithi02683976-12 from 2252800 to 3014656 tid 1 2024-09-13T14:21:45.973 INFO:tasks.rados.rados.0.smithi026.stdout:850: writing smithi02683976-12 from 3014656 to 3612672 tid 2 2024-09-13T14:21:45.976 INFO:tasks.rados.rados.0.smithi026.stdout:850: writing smithi02683976-12 from 3612672 to 4284416 tid 3 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:850: writing smithi02683976-12 from 4284416 to 4620288 tid 4 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:851: read oid 21 snap 68 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:851: expect (ObjNum 226 snap 55 seq_num 226) 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 279 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 1 to smithi02683976-12 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 2 to smithi02683976-12 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:852: read oid 34 snap 72 2024-09-13T14:21:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:852: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:45.980 INFO:tasks.rados.rados.0.smithi026.stdout:849: done (3 left) 2024-09-13T14:21:45.980 INFO:tasks.rados.rados.0.smithi026.stdout:853: rollback oid 1 current snap is 73 2024-09-13T14:21:45.980 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 68 2024-09-13T14:21:45.980 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 73 2024-09-13T14:21:45.980 INFO:tasks.rados.rados.0.smithi026.stdout:854: seq_num 285 ranges {6012928=778240,6791168=475136,7266304=475136,7741440=655360} 2024-09-13T14:21:46.017 INFO:tasks.rados.rados.0.smithi026.stdout:854: writing smithi02683976-13 from 6012928 to 6791168 tid 1 2024-09-13T14:21:46.021 INFO:tasks.rados.rados.0.smithi026.stdout:854: writing smithi02683976-13 from 6791168 to 7266304 tid 2 2024-09-13T14:21:46.024 INFO:tasks.rados.rados.0.smithi026.stdout:854: writing smithi02683976-13 from 7266304 to 7741440 tid 3 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:854: writing smithi02683976-13 from 7741440 to 8396800 tid 4 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:855: rollback oid 37 current snap is 73 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 68 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:853: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 3 to smithi02683976-12 2024-09-13T14:21:46.029 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 4 to smithi02683976-12 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:853: done (5 left) 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:856: read oid 22 snap -1 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:856: expect deleted 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:857: snap_create 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 5 to smithi02683976-12 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:850: finishing write tid 6 to smithi02683976-12 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 184 (ObjNum 284 snap 73 seq_num 284) dirty exists 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:850: left oid 12 (ObjNum 284 snap 73 seq_num 284) 2024-09-13T14:21:46.030 INFO:tasks.rados.rados.0.smithi026.stdout:851: expect (ObjNum 226 snap 55 seq_num 226) 2024-09-13T14:21:46.074 INFO:tasks.rados.rados.0.smithi026.stdout:852: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:21:46.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:21:46.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:46.144 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:45 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:21:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:21:46.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:21:46.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:21:46.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:45 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:46.680 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 1 to smithi02683976-13 2024-09-13T14:21:46.680 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 2 to smithi02683976-13 2024-09-13T14:21:46.680 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 3 to smithi02683976-13 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:855: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:855: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 4 to smithi02683976-13 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:855: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 282 (ObjNum 200 snap 48 seq_num 200) dirty exists 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 5 to smithi02683976-13 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:854: finishing write tid 6 to smithi02683976-13 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 490 (ObjNum 285 snap 73 seq_num 285) dirty exists 2024-09-13T14:21:46.681 INFO:tasks.rados.rados.0.smithi026.stdout:854: left oid 13 (ObjNum 285 snap 73 seq_num 285) 2024-09-13T14:21:46.897 INFO:tasks.rados.rados.0.smithi026.stdout:850: done (6 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:851: done (5 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:852: done (4 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:854: done (3 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:855: done (2 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:856: done (1 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:857: done (0 left) 2024-09-13T14:21:46.898 INFO:tasks.rados.rados.0.smithi026.stdout:858: snap_remove snap 72 2024-09-13T14:21:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[27770]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[27770]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[27770]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:21:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[27770]: pgmap v93: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[39182]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[39182]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[39182]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:21:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[39182]: pgmap v93: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:46 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:46 smithi145 ceph-mon[31057]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:46 smithi145 ceph-mon[31057]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:21:47.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:46 smithi145 ceph-mon[31057]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:21:47.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:46 smithi145 ceph-mon[31057]: pgmap v93: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-09-13T14:21:47.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:46 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:47.900 INFO:tasks.rados.rados.0.smithi026.stdout:858: done (0 left) 2024-09-13T14:21:47.900 INFO:tasks.rados.rados.0.smithi026.stdout:859: rmattr oid 41 current snap is 74 2024-09-13T14:21:47.900 INFO:tasks.rados.rados.0.smithi026.stdout:859: done (0 left) 2024-09-13T14:21:47.900 INFO:tasks.rados.rados.0.smithi026.stdout:860: snap_remove snap 73 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: Upgrade: Updating mgr.y 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[39182]: Deploying daemon mgr.y on smithi026 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: Upgrade: Updating mgr.y 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:47 smithi026 ceph-mon[27770]: Deploying daemon mgr.y on smithi026 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: Upgrade: Updating mgr.y 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:21:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:21:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:47 smithi145 ceph-mon[31057]: Deploying daemon mgr.y on smithi026 2024-09-13T14:21:48.756 INFO:tasks.rados.rados.0.smithi026.stdout:860: done (0 left) 2024-09-13T14:21:48.757 INFO:tasks.rados.rados.0.smithi026.stdout:861: setattr oid 11 current snap is 74 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:862: rmattr oid 31 current snap is 74 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:863: rollback oid 1 current snap is 74 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 68 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:864: rollback oid 27 current snap is 74 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 68 2024-09-13T14:21:48.758 INFO:tasks.rados.rados.0.smithi026.stdout:865: snap_create 2024-09-13T14:21:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:863: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:21:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:21:48.764 INFO:tasks.rados.rados.0.smithi026.stdout:864: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:21:48.764 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 0 (ObjNum 161 snap 0 seq_num 518955072) dirty dne 2024-09-13T14:21:48.766 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 459 (ObjNum 268 snap 69 seq_num 268) dirty exists 2024-09-13T14:21:48.767 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 460 (ObjNum 256 snap 66 seq_num 256) dirty exists 2024-09-13T14:21:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[27770]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:21:49.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[27770]: pgmap v96: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:49.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[27770]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:21:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[39182]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:21:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[39182]: pgmap v96: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:48 smithi026 ceph-mon[39182]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:21:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:48 smithi145 ceph-mon[31057]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:21:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:48 smithi145 ceph-mon[31057]: pgmap v96: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:48 smithi145 ceph-mon[31057]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:21:49.469 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:49 smithi026 systemd[1]: Stopping Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:861: done (4 left) 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:862: done (3 left) 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:863: done (2 left) 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:864: done (1 left) 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:865: done (0 left) 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:866: rollback oid 43 current snap is 75 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 43 to 68 2024-09-13T14:21:49.757 INFO:tasks.rados.rados.0.smithi026.stdout:867: snap_create 2024-09-13T14:21:49.760 INFO:tasks.rados.rados.0.smithi026.stdout:866: finishing rollback tid 0 to smithi02683976-43 2024-09-13T14:21:49.761 INFO:tasks.rados.rados.0.smithi026.stdout:866: finishing rollback tid 1 to smithi02683976-43 2024-09-13T14:21:49.761 INFO:tasks.rados.rados.0.smithi026.stdout:866: finishing rollback tid 2 to smithi02683976-43 2024-09-13T14:21:49.761 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 496 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:49.770 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:49 smithi026 podman[87979]: 2024-09-13 14:21:49.468392877 +0000 UTC m=+0.155390009 container died 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:21:50.096 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:49 smithi026 podman[87979]: 2024-09-13 14:21:49.843635721 +0000 UTC m=+0.530632842 container cleanup 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T14:21:50.096 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:49 smithi026 bash[87979]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:21:50.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:49 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:21:49] "GET /metrics HTTP/1.1" 200 36133 "" "Prometheus/2.51.0" 2024-09-13T14:21:50.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 podman[87997]: 2024-09-13 14:21:50.09547385 +0000 UTC m=+0.630933130 container remove 90a812e595d2f6f659109fa6a56eb42b9437e198f0ce4ee3231f77f906f43542 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T14:21:50.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:21:50.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T14:21:50.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 systemd[1]: Stopped Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:50.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service: Consumed 45.401s CPU time. 2024-09-13T14:21:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:50.758 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 systemd[1]: Starting Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:50.758 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 podman[88132]: 2024-09-13 14:21:50.547929567 +0000 UTC m=+0.016378132 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:50.758 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 podman[88132]: 2024-09-13 14:21:50.663361544 +0000 UTC m=+0.131810109 container create 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True) 2024-09-13T14:21:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:866: done (1 left) 2024-09-13T14:21:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:867: done (0 left) 2024-09-13T14:21:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:868: rmattr oid 29 current snap is 76 2024-09-13T14:21:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:868: done (0 left) 2024-09-13T14:21:50.761 INFO:tasks.rados.rados.0.smithi026.stdout:869: snap_create 2024-09-13T14:21:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:50 smithi026 ceph-mon[39182]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:21:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:50 smithi026 ceph-mon[39182]: pgmap v99: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:50 smithi026 ceph-mon[27770]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:50 smithi026 ceph-mon[27770]: pgmap v99: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 podman[88132]: 2024-09-13 14:21:50.899646596 +0000 UTC m=+0.368095167 container init 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 podman[88132]: 2024-09-13 14:21:50.904588966 +0000 UTC m=+0.373037522 container start 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 ceph-mgr[88170]: -- 172.21.15.26:0/752981041 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563dbcdf90e0 con 0x563dbcdd6800 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:50 smithi026 bash[88132]: 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 2024-09-13T14:21:51.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:51 smithi026 systemd[1]: Started Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:50 smithi145 ceph-mon[31057]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:21:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:50 smithi145 ceph-mon[31057]: pgmap v99: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-13T14:21:51.763 INFO:tasks.rados.rados.0.smithi026.stdout:869: done (0 left) 2024-09-13T14:21:51.763 INFO:tasks.rados.rados.0.smithi026.stdout:870: delete oid 33 current snap is 77 2024-09-13T14:21:51.771 INFO:tasks.rados.rados.0.smithi026.stdout:870: done (0 left) 2024-09-13T14:21:51.771 INFO:tasks.rados.rados.0.smithi026.stdout:871: setattr oid 48 current snap is 77 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 77 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:872: seq_num 287 ranges {0=8192} 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:872: writing smithi02683976-2 from 0 to 8192 tid 1 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:873: read oid 37 snap -1 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:873: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:874: copy_from oid 40 from oid 33 current snap is 77 2024-09-13T14:21:51.774 INFO:tasks.rados.rados.0.smithi026.stdout:875: snap_remove snap 68 2024-09-13T14:21:52.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:21:52.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:51 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:21:52.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:52.116 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.116 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:51 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:52.780 INFO:tasks.rados.rados.0.smithi026.stdout:875: done (4 left) 2024-09-13T14:21:52.780 INFO:tasks.rados.rados.0.smithi026.stdout:876: setattr oid 18 current snap is 77 2024-09-13T14:21:52.780 INFO:tasks.rados.rados.0.smithi026.stdout:877: snap_create 2024-09-13T14:21:52.780 INFO:tasks.rados.rados.0.smithi026.stdout:872: finishing write tid 1 to smithi02683976-2 2024-09-13T14:21:52.780 INFO:tasks.rados.rados.0.smithi026.stdout:872: finishing write tid 2 to smithi02683976-2 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:874: finishing copy_from to smithi02683976-40 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:874: got expected ENOENT (src dne) 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:874: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:872: finishing write tid 3 to smithi02683976-2 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 246 (ObjNum 287 snap 77 seq_num 287) dirty exists 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:872: left oid 2 (ObjNum 287 snap 77 seq_num 287) 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 137 (ObjNum 280 snap 71 seq_num 280) dirty exists 2024-09-13T14:21:52.781 INFO:tasks.rados.rados.0.smithi026.stdout:873: expect (ObjNum 200 snap 48 seq_num 200) 2024-09-13T14:21:52.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:52 smithi026 ceph-mon[27770]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:21:52.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:52 smithi026 ceph-mon[27770]: pgmap v102: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:52.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:52 smithi026 ceph-mon[39182]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:21:52.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:52 smithi026 ceph-mon[39182]: pgmap v102: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.096 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 384 (ObjNum 77 snap 10 seq_num 77) dirty exists 2024-09-13T14:21:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:52 smithi145 ceph-mon[31057]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:21:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:52 smithi145 ceph-mon[31057]: pgmap v102: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.776 INFO:tasks.rados.rados.0.smithi026.stdout:871: done (5 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:872: done (4 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:873: done (3 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:874: done (2 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:876: done (1 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:877: done (0 left) 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 78 2024-09-13T14:21:53.777 INFO:tasks.rados.rados.0.smithi026.stdout:878: seq_num 288 ranges {5423104=24576} 2024-09-13T14:21:53.806 INFO:tasks.rados.rados.0.smithi026.stdout:878: writing smithi02683976-37 from 5423104 to 5447680 tid 1 2024-09-13T14:21:53.806 INFO:tasks.rados.rados.0.smithi026.stdout:879: snap_create 2024-09-13T14:21:53.811 INFO:tasks.rados.rados.0.smithi026.stdout:878: finishing write tid 1 to smithi02683976-37 2024-09-13T14:21:53.811 INFO:tasks.rados.rados.0.smithi026.stdout:878: finishing write tid 2 to smithi02683976-37 2024-09-13T14:21:53.812 INFO:tasks.rados.rados.0.smithi026.stdout:878: finishing write tid 3 to smithi02683976-37 2024-09-13T14:21:53.812 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 284 (ObjNum 288 snap 78 seq_num 288) dirty exists 2024-09-13T14:21:53.812 INFO:tasks.rados.rados.0.smithi026.stdout:878: left oid 37 (ObjNum 288 snap 78 seq_num 288) 2024-09-13T14:21:53.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:53 smithi145 ceph-mon[31057]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:21:54.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:53 smithi026 ceph-mon[39182]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:21:54.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:53 smithi026 ceph-mon[27770]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:21:54.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:54.782 INFO:tasks.rados.rados.0.smithi026.stdout:878: done (1 left) 2024-09-13T14:21:54.782 INFO:tasks.rados.rados.0.smithi026.stdout:879: done (0 left) 2024-09-13T14:21:54.783 INFO:tasks.rados.rados.0.smithi026.stdout:880: read oid 11 snap 74 2024-09-13T14:21:54.783 INFO:tasks.rados.rados.0.smithi026.stdout:880: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:21:54.783 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 79 2024-09-13T14:21:54.783 INFO:tasks.rados.rados.0.smithi026.stdout:881: seq_num 289 ranges {1392640=40960} 2024-09-13T14:21:54.790 INFO:tasks.rados.rados.0.smithi026.stdout:881: writing smithi02683976-21 from 1392640 to 1433600 tid 1 2024-09-13T14:21:54.790 INFO:tasks.rados.rados.0.smithi026.stdout:882: delete oid 47 current snap is 79 2024-09-13T14:21:54.801 INFO:tasks.rados.rados.0.smithi026.stdout:880: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:21:54.985 INFO:tasks.rados.rados.0.smithi026.stdout:881: finishing write tid 1 to smithi02683976-21 2024-09-13T14:21:54.985 INFO:tasks.rados.rados.0.smithi026.stdout:881: finishing write tid 2 to smithi02683976-21 2024-09-13T14:21:54.985 INFO:tasks.rados.rados.0.smithi026.stdout:881: finishing write tid 3 to smithi02683976-21 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 471 (ObjNum 289 snap 79 seq_num 289) dirty exists 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:881: left oid 21 (ObjNum 289 snap 79 seq_num 289) 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:880: done (2 left) 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:881: done (1 left) 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:882: done (0 left) 2024-09-13T14:21:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:883: snap_create 2024-09-13T14:21:55.016 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:21:55.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:54 smithi026 ceph-mon[39182]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:21:55.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:54 smithi026 ceph-mon[39182]: pgmap v105: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:55.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:54 smithi026 ceph-mon[27770]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:21:55.016 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:54 smithi026 ceph-mon[27770]: pgmap v105: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:54 smithi145 ceph-mon[31057]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:21:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:54 smithi145 ceph-mon[31057]: pgmap v105: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (39s) 0s ago 5m 17.5M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 starting - - - - 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (6m) 0s ago 6m 13.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (6m) 62s ago 6m 19.1M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283 running (78s) 62s ago 10m 495M - 19.3.0-4697-gd58f7306 fec07016a2b0 a4ae2b625819 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (5s) 0s ago 11m 69.6M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (11m) 0s ago 12m 54.4M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (10m) 62s ago 10m 47.0M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:21:55.627 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (10m) 0s ago 10m 43.3M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (33s) 0s ago 4m 7625k - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 starting - - - - 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (9m) 0s ago 9m 136M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (9m) 0s ago 9m 122M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (9m) 0s ago 9m 123M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (8m) 0s ago 8m 80.0M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (8m) 62s ago 8m 119M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (7m) 62s ago 7m 99.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (7m) 62s ago 7m 73.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (7m) 62s ago 7m 76.2M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:21:55.628 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 starting - - - - 2024-09-13T14:21:55.784 INFO:tasks.rados.rados.0.smithi026.stdout:883: done (0 left) 2024-09-13T14:21:55.784 INFO:tasks.rados.rados.0.smithi026.stdout:884: setattr oid 39 current snap is 80 2024-09-13T14:21:55.784 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:55.784 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:885: read oid 37 snap -1 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:885: expect (ObjNum 288 snap 78 seq_num 288) 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:886: read oid 25 snap -1 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:886: expect (ObjNum 218 snap 53 seq_num 218) 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:887: read oid 43 snap -1 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:887: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:888: rmattr oid 12 current snap is 80 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:888: done (4 left) 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 80 2024-09-13T14:21:55.786 INFO:tasks.rados.rados.0.smithi026.stdout:889: seq_num 291 ranges {303104=81920} 2024-09-13T14:21:55.789 INFO:tasks.rados.rados.0.smithi026.stdout:889: writing smithi02683976-9 from 303104 to 385024 tid 1 2024-09-13T14:21:55.789 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 80 2024-09-13T14:21:55.789 INFO:tasks.rados.rados.0.smithi026.stdout:890: seq_num 292 ranges {6807552=98304} 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:890: writing smithi02683976-26 from 6807552 to 6905856 tid 1 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:891: rmattr oid 31 current snap is 80 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:889: finishing write tid 1 to smithi02683976-9 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:889: finishing write tid 2 to smithi02683976-9 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:892: snap_create 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:889: finishing write tid 3 to smithi02683976-9 2024-09-13T14:21:55.832 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 474 (ObjNum 291 snap 80 seq_num 291) dirty exists 2024-09-13T14:21:55.833 INFO:tasks.rados.rados.0.smithi026.stdout:889: left oid 9 (ObjNum 291 snap 80 seq_num 291) 2024-09-13T14:21:55.833 INFO:tasks.rados.rados.0.smithi026.stdout:887: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:55.918 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 250 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:21:55.918 INFO:tasks.rados.rados.0.smithi026.stdout:886: expect (ObjNum 218 snap 53 seq_num 218) 2024-09-13T14:21:55.971 INFO:tasks.rados.rados.0.smithi026.stdout:885: expect (ObjNum 288 snap 78 seq_num 288) 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.098 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:21:56.098 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:21:56.098 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:21:56.099 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:21:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 14, 2024-09-13T14:21:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:21:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:21:56.100 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:55 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:55 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 476 (ObjNum 256 snap 66 seq_num 256) dirty exists 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:890: finishing write tid 1 to smithi02683976-26 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:890: finishing write tid 2 to smithi02683976-26 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:890: finishing write tid 3 to smithi02683976-26 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 195 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:21:56.381 INFO:tasks.rados.rados.0.smithi026.stdout:890: left oid 26 (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:21:56.435 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:21:56.435 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:21:56.435 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:21:56.435 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/20 daemons upgraded", 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:21:56.436 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:21:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:56] ENGINE Bus STARTING 2024-09-13T14:21:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: CherryPy Checker: 2024-09-13T14:21:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: The Application mounted at '' has an empty config. 2024-09-13T14:21:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:21:56.793 INFO:tasks.rados.rados.0.smithi026.stdout:884: done (7 left) 2024-09-13T14:21:56.793 INFO:tasks.rados.rados.0.smithi026.stdout:885: done (6 left) 2024-09-13T14:21:56.793 INFO:tasks.rados.rados.0.smithi026.stdout:886: done (5 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:887: done (4 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:889: done (3 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:890: done (2 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:891: done (1 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:892: done (0 left) 2024-09-13T14:21:56.794 INFO:tasks.rados.rados.0.smithi026.stdout:893: snap_create 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/788734822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: pgmap v108: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:57.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: Standby manager daemon y restarted 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: Standby manager daemon y started 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/788734822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: pgmap v108: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: Standby manager daemon y restarted 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: Standby manager daemon y started 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:56] ENGINE Serving on http://:::9283 2024-09-13T14:21:57.046 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:56] ENGINE Bus STARTED 2024-09-13T14:21:57.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:21:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/788734822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: pgmap v108: 121 pgs: 121 active+clean; 147 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: Standby manager daemon y restarted 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: Standby manager daemon y started 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:21:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:56 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:57.799 INFO:tasks.rados.rados.0.smithi026.stdout:893: done (0 left) 2024-09-13T14:21:57.799 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 82 2024-09-13T14:21:57.799 INFO:tasks.rados.rados.0.smithi026.stdout:894: seq_num 293 ranges {0=114688} 2024-09-13T14:21:57.800 INFO:tasks.rados.rados.0.smithi026.stdout:894: writing smithi02683976-17 from 0 to 114688 tid 1 2024-09-13T14:21:57.800 INFO:tasks.rados.rados.0.smithi026.stdout:895: delete oid 15 current snap is 82 2024-09-13T14:21:57.805 INFO:tasks.rados.rados.0.smithi026.stdout:895: done (1 left) 2024-09-13T14:21:57.805 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 82 2024-09-13T14:21:57.805 INFO:tasks.rados.rados.0.smithi026.stdout:896: seq_num 295 ranges {0=147456} 2024-09-13T14:21:57.805 INFO:tasks.rados.rados.0.smithi026.stdout:896: writing smithi02683976-27 from 0 to 147456 tid 1 2024-09-13T14:21:57.806 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 82 2024-09-13T14:21:57.806 INFO:tasks.rados.rados.0.smithi026.stdout:897: seq_num 296 ranges {4333568=163840} 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:897: writing smithi02683976-10 from 4333568 to 4497408 tid 1 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:898: delete oid 32 current snap is 82 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:894: finishing write tid 1 to smithi02683976-17 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:894: finishing write tid 2 to smithi02683976-17 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:894: finishing write tid 3 to smithi02683976-17 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 379 (ObjNum 293 snap 82 seq_num 293) dirty exists 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:894: left oid 17 (ObjNum 293 snap 82 seq_num 293) 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:896: finishing write tid 1 to smithi02683976-27 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:896: finishing write tid 2 to smithi02683976-27 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:896: finishing write tid 3 to smithi02683976-27 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 464 (ObjNum 295 snap 82 seq_num 295) dirty exists 2024-09-13T14:21:57.831 INFO:tasks.rados.rados.0.smithi026.stdout:896: left oid 27 (ObjNum 295 snap 82 seq_num 295) 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:897: finishing write tid 1 to smithi02683976-10 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:897: finishing write tid 2 to smithi02683976-10 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:894: done (3 left) 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:896: done (2 left) 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:898: done (1 left) 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:899: read oid 41 snap -1 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:899: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 82 2024-09-13T14:21:57.834 INFO:tasks.rados.rados.0.smithi026.stdout:900: seq_num 298 ranges {0=196608} 2024-09-13T14:21:57.835 INFO:tasks.rados.rados.0.smithi026.stdout:900: writing smithi02683976-29 from 0 to 196608 tid 1 2024-09-13T14:21:57.835 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 82 2024-09-13T14:21:57.835 INFO:tasks.rados.rados.0.smithi026.stdout:901: seq_num 299 ranges {1867776=212992} 2024-09-13T14:21:57.847 INFO:tasks.rados.rados.0.smithi026.stdout:901: writing smithi02683976-39 from 1867776 to 2080768 tid 1 2024-09-13T14:21:57.847 INFO:tasks.rados.rados.0.smithi026.stdout:902: delete oid 46 current snap is 82 2024-09-13T14:21:57.848 INFO:tasks.rados.rados.0.smithi026.stdout:897: finishing write tid 3 to smithi02683976-10 2024-09-13T14:21:57.848 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 264 (ObjNum 296 snap 82 seq_num 296) dirty exists 2024-09-13T14:21:57.848 INFO:tasks.rados.rados.0.smithi026.stdout:897: left oid 10 (ObjNum 296 snap 82 seq_num 296) 2024-09-13T14:21:57.848 INFO:tasks.rados.rados.0.smithi026.stdout:900: finishing write tid 1 to smithi02683976-29 2024-09-13T14:21:57.848 INFO:tasks.rados.rados.0.smithi026.stdout:900: finishing write tid 2 to smithi02683976-29 2024-09-13T14:21:57.849 INFO:tasks.rados.rados.0.smithi026.stdout:900: finishing write tid 3 to smithi02683976-29 2024-09-13T14:21:57.849 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 506 (ObjNum 298 snap 82 seq_num 298) dirty exists 2024-09-13T14:21:57.849 INFO:tasks.rados.rados.0.smithi026.stdout:900: left oid 29 (ObjNum 298 snap 82 seq_num 298) 2024-09-13T14:21:57.849 INFO:tasks.rados.rados.0.smithi026.stdout:899: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:901: finishing write tid 1 to smithi02683976-39 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:901: finishing write tid 2 to smithi02683976-39 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:901: finishing write tid 3 to smithi02683976-39 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 253 (ObjNum 299 snap 82 seq_num 299) dirty exists 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:901: left oid 39 (ObjNum 299 snap 82 seq_num 299) 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:897: done (4 left) 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:899: done (3 left) 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:900: done (2 left) 2024-09-13T14:21:57.931 INFO:tasks.rados.rados.0.smithi026.stdout:901: done (1 left) 2024-09-13T14:21:57.932 INFO:tasks.rados.rados.0.smithi026.stdout:902: done (0 left) 2024-09-13T14:21:57.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 82 2024-09-13T14:21:57.932 INFO:tasks.rados.rados.0.smithi026.stdout:903: seq_num 301 ranges {0=245760} 2024-09-13T14:21:57.933 INFO:tasks.rados.rados.0.smithi026.stdout:903: writing smithi02683976-50 from 0 to 245760 tid 1 2024-09-13T14:21:57.933 INFO:tasks.rados.rados.0.smithi026.stdout:904: copy_from oid 44 from oid 47 current snap is 82 2024-09-13T14:21:57.933 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 82 2024-09-13T14:21:57.933 INFO:tasks.rados.rados.0.smithi026.stdout:905: seq_num 302 ranges {0=262144} 2024-09-13T14:21:57.934 INFO:tasks.rados.rados.0.smithi026.stdout:905: writing smithi02683976-5 from 0 to 262144 tid 1 2024-09-13T14:21:57.934 INFO:tasks.rados.rados.0.smithi026.stdout:906: delete oid 18 current snap is 82 2024-09-13T14:21:57.934 INFO:tasks.rados.rados.0.smithi026.stdout:904: finishing copy_from to smithi02683976-44 2024-09-13T14:21:57.935 INFO:tasks.rados.rados.0.smithi026.stdout:904: got expected ENOENT (src dne) 2024-09-13T14:21:57.935 INFO:tasks.rados.rados.0.smithi026.stdout:904: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:21:57.935 INFO:tasks.rados.rados.0.smithi026.stdout:903: finishing write tid 1 to smithi02683976-50 2024-09-13T14:21:57.936 INFO:tasks.rados.rados.0.smithi026.stdout:903: finishing write tid 2 to smithi02683976-50 2024-09-13T14:21:57.936 INFO:tasks.rados.rados.0.smithi026.stdout:905: finishing write tid 1 to smithi02683976-5 2024-09-13T14:21:57.937 INFO:tasks.rados.rados.0.smithi026.stdout:904: done (3 left) 2024-09-13T14:21:57.937 INFO:tasks.rados.rados.0.smithi026.stdout:906: done (2 left) 2024-09-13T14:21:57.937 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 82 2024-09-13T14:21:57.937 INFO:tasks.rados.rados.0.smithi026.stdout:907: seq_num 304 ranges {0=294912} 2024-09-13T14:21:57.939 INFO:tasks.rados.rados.0.smithi026.stdout:907: writing smithi02683976-33 from 0 to 294912 tid 1 2024-09-13T14:21:57.939 INFO:tasks.rados.rados.0.smithi026.stdout:908: snap_create 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:905: finishing write tid 2 to smithi02683976-5 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:903: finishing write tid 3 to smithi02683976-50 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 478 (ObjNum 301 snap 82 seq_num 301) dirty exists 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:903: left oid 50 (ObjNum 301 snap 82 seq_num 301) 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:905: finishing write tid 3 to smithi02683976-5 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 466 (ObjNum 302 snap 82 seq_num 302) dirty exists 2024-09-13T14:21:57.940 INFO:tasks.rados.rados.0.smithi026.stdout:905: left oid 5 (ObjNum 302 snap 82 seq_num 302) 2024-09-13T14:21:57.941 INFO:tasks.rados.rados.0.smithi026.stdout:907: finishing write tid 1 to smithi02683976-33 2024-09-13T14:21:57.941 INFO:tasks.rados.rados.0.smithi026.stdout:907: finishing write tid 2 to smithi02683976-33 2024-09-13T14:21:57.942 INFO:tasks.rados.rados.0.smithi026.stdout:907: finishing write tid 3 to smithi02683976-33 2024-09-13T14:21:57.942 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 468 (ObjNum 304 snap 82 seq_num 304) dirty exists 2024-09-13T14:21:57.942 INFO:tasks.rados.rados.0.smithi026.stdout:907: left oid 33 (ObjNum 304 snap 82 seq_num 304) 2024-09-13T14:21:58.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:57 smithi026 ceph-mon[27770]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:21:58.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:57 smithi026 ceph-mon[27770]: mgrmap e29: x(active, since 70s), standbys: y 2024-09-13T14:21:58.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:57 smithi026 ceph-mon[39182]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:21:58.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:57 smithi026 ceph-mon[39182]: mgrmap e29: x(active, since 70s), standbys: y 2024-09-13T14:21:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:57 smithi145 ceph-mon[31057]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:21:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:57 smithi145 ceph-mon[31057]: mgrmap e29: x(active, since 70s), standbys: y 2024-09-13T14:21:58.757 INFO:tasks.rados.rados.0.smithi026.stdout:903: done (3 left) 2024-09-13T14:21:58.757 INFO:tasks.rados.rados.0.smithi026.stdout:905: done (2 left) 2024-09-13T14:21:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:907: done (1 left) 2024-09-13T14:21:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:908: done (0 left) 2024-09-13T14:21:58.758 INFO:tasks.rados.rados.0.smithi026.stdout:909: snap_remove snap 77 2024-09-13T14:21:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[39182]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:21:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[39182]: pgmap v111: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 60 KiB/s wr, 17 op/s 2024-09-13T14:21:59.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[39182]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:21:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[27770]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:21:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[27770]: pgmap v111: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 60 KiB/s wr, 17 op/s 2024-09-13T14:21:59.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:21:58 smithi026 ceph-mon[27770]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:21:59.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:58 smithi145 ceph-mon[31057]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:21:59.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:58 smithi145 ceph-mon[31057]: pgmap v111: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 60 KiB/s wr, 17 op/s 2024-09-13T14:21:59.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:21:58 smithi145 ceph-mon[31057]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:21:59.355 INFO:tasks.rados.rados.0.smithi026.stdout:909: done (0 left) 2024-09-13T14:21:59.355 INFO:tasks.rados.rados.0.smithi026.stdout:910: rmattr oid 46 current snap is 83 2024-09-13T14:21:59.355 INFO:tasks.rados.rados.0.smithi026.stdout:910: done (0 left) 2024-09-13T14:21:59.355 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 83 2024-09-13T14:21:59.356 INFO:tasks.rados.rados.0.smithi026.stdout:911: seq_num 305 ranges {1433600=311296} 2024-09-13T14:21:59.365 INFO:tasks.rados.rados.0.smithi026.stdout:911: writing smithi02683976-21 from 1433600 to 1744896 tid 1 2024-09-13T14:21:59.365 INFO:tasks.rados.rados.0.smithi026.stdout:912: delete oid 39 current snap is 83 2024-09-13T14:21:59.369 INFO:tasks.rados.rados.0.smithi026.stdout:912: done (1 left) 2024-09-13T14:21:59.369 INFO:tasks.rados.rados.0.smithi026.stdout:913: read oid 49 snap -1 2024-09-13T14:21:59.369 INFO:tasks.rados.rados.0.smithi026.stdout:913: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:21:59.370 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 83 2024-09-13T14:21:59.370 INFO:tasks.rados.rados.0.smithi026.stdout:914: seq_num 307 ranges {4825088=344064} 2024-09-13T14:21:59.399 INFO:tasks.rados.rados.0.smithi026.stdout:914: writing smithi02683976-28 from 4825088 to 5169152 tid 1 2024-09-13T14:21:59.399 INFO:tasks.rados.rados.0.smithi026.stdout:915: read oid 39 snap -1 2024-09-13T14:21:59.399 INFO:tasks.rados.rados.0.smithi026.stdout:915: expect deleted 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:916: rmattr oid 12 current snap is 83 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:916: done (4 left) 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:917: delete oid 3 current snap is 83 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:911: finishing write tid 1 to smithi02683976-21 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:911: finishing write tid 2 to smithi02683976-21 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:911: finishing write tid 3 to smithi02683976-21 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 481 (ObjNum 305 snap 83 seq_num 305) dirty exists 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:911: left oid 21 (ObjNum 305 snap 83 seq_num 305) 2024-09-13T14:21:59.400 INFO:tasks.rados.rados.0.smithi026.stdout:913: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:21:59.537 INFO:tasks.rados.rados.0.smithi026.stdout:914: finishing write tid 1 to smithi02683976-28 2024-09-13T14:21:59.537 INFO:tasks.rados.rados.0.smithi026.stdout:914: finishing write tid 2 to smithi02683976-28 2024-09-13T14:21:59.537 INFO:tasks.rados.rados.0.smithi026.stdout:914: finishing write tid 3 to smithi02683976-28 2024-09-13T14:21:59.537 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 269 (ObjNum 307 snap 83 seq_num 307) dirty exists 2024-09-13T14:21:59.537 INFO:tasks.rados.rados.0.smithi026.stdout:914: left oid 28 (ObjNum 307 snap 83 seq_num 307) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:911: done (4 left) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:913: done (3 left) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:914: done (2 left) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:915: done (1 left) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:917: done (0 left) 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:918: copy_from oid 50 from oid 30 current snap is 83 2024-09-13T14:21:59.538 INFO:tasks.rados.rados.0.smithi026.stdout:919: snap_remove snap 81 2024-09-13T14:21:59.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:59] ENGINE Bus STOPPING 2024-09-13T14:21:59.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:59 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ignoring --setuser ceph since I am not root 2024-09-13T14:21:59.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:59 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ignoring --setgroup ceph since I am not root 2024-09-13T14:21:59.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:21:59 smithi145 ceph-mgr[66802]: -- 172.21.15.145:0/3653962293 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bfb05f50e0 con 0x55bfb05d3400 2024-09-13T14:22:00.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:22:00.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:21:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:21:59] ENGINE Bus STOPPED 2024-09-13T14:22:00.356 INFO:tasks.rados.rados.0.smithi026.stdout:918: finishing copy_from to smithi02683976-50 2024-09-13T14:22:00.357 INFO:tasks.rados.rados.0.smithi026.stdout:918: got expected ENOENT (src dne) 2024-09-13T14:22:00.357 INFO:tasks.rados.rados.0.smithi026.stdout:918: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:22:00.357 INFO:tasks.rados.rados.0.smithi026.stdout:918: done (1 left) 2024-09-13T14:22:00.357 INFO:tasks.rados.rados.0.smithi026.stdout:919: done (0 left) 2024-09-13T14:22:00.357 INFO:tasks.rados.rados.0.smithi026.stdout:920: snap_create 2024-09-13T14:22:00.475 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.314+0000 7fc238f17640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.314+0000 7fc238f17640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.314+0000 7fc238f17640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.314+0000 7fc238f17640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.314+0000 7fc238f17640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.315+0000 7fc23af1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.315+0000 7fc23af1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.315+0000 7fc23af1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.315+0000 7fc23af1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:00.315+0000 7fc23af1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:00.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:00] ENGINE Bus STARTING 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: Activating manager daemon y 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:22:00.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.24763 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: mgrmap e30: y(active, starting, since 0.0063326s) 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:00.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[39182]: Manager daemon y is now available 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:00.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: Activating manager daemon y 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.24763 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: mgrmap e30: y(active, starting, since 0.0063326s) 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:00.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:00.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-mon[27770]: Manager daemon y is now available 2024-09-13T14:22:00.483 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 172.21.15.145:0/1406027017' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:22:00.483 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: Activating manager daemon y 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.24763 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: mgrmap e30: y(active, starting, since 0.0063326s) 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:00.484 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:00.485 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:00 smithi145 ceph-mon[31057]: Manager daemon y is now available 2024-09-13T14:22:00.915 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:00] ENGINE Serving on http://:::9283 2024-09-13T14:22:00.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:00] ENGINE Bus STARTED 2024-09-13T14:22:01.357 INFO:tasks.rados.rados.0.smithi026.stdout:920: done (0 left) 2024-09-13T14:22:01.357 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 84 2024-09-13T14:22:01.357 INFO:tasks.rados.rados.0.smithi026.stdout:921: seq_num 309 ranges {1744896=376832} 2024-09-13T14:22:01.370 INFO:tasks.rados.rados.0.smithi026.stdout:921: writing smithi02683976-21 from 1744896 to 2121728 tid 1 2024-09-13T14:22:01.371 INFO:tasks.rados.rados.0.smithi026.stdout:922: rollback oid 42 current snap is 84 2024-09-13T14:22:01.371 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 78 2024-09-13T14:22:01.371 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 84 2024-09-13T14:22:01.371 INFO:tasks.rados.rados.0.smithi026.stdout:923: seq_num 310 ranges {0=401408} 2024-09-13T14:22:01.374 INFO:tasks.rados.rados.0.smithi026.stdout:923: writing smithi02683976-47 from 0 to 401408 tid 1 2024-09-13T14:22:01.374 INFO:tasks.rados.rados.0.smithi026.stdout:924: copy_from oid 13 from oid 7 current snap is 84 2024-09-13T14:22:01.374 INFO:tasks.rados.rados.0.smithi026.stdout:925: snap_create 2024-09-13T14:22:01.377 INFO:tasks.rados.rados.0.smithi026.stdout:921: finishing write tid 1 to smithi02683976-21 2024-09-13T14:22:01.377 INFO:tasks.rados.rados.0.smithi026.stdout:924: finishing copy_from to smithi02683976-13 2024-09-13T14:22:01.377 INFO:tasks.rados.rados.0.smithi026.stdout:924: got expected ENOENT (src dne) 2024-09-13T14:22:01.377 INFO:tasks.rados.rados.0.smithi026.stdout:921: finishing write tid 2 to smithi02683976-21 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:924: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:922: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:922: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:921: finishing write tid 3 to smithi02683976-21 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 492 (ObjNum 309 snap 84 seq_num 309) dirty exists 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:921: left oid 21 (ObjNum 309 snap 84 seq_num 309) 2024-09-13T14:22:01.378 INFO:tasks.rados.rados.0.smithi026.stdout:922: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:22:01.379 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 287 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:22:01.379 INFO:tasks.rados.rados.0.smithi026.stdout:923: finishing write tid 1 to smithi02683976-47 2024-09-13T14:22:01.379 INFO:tasks.rados.rados.0.smithi026.stdout:923: finishing write tid 2 to smithi02683976-47 2024-09-13T14:22:01.381 INFO:tasks.rados.rados.0.smithi026.stdout:923: finishing write tid 3 to smithi02683976-47 2024-09-13T14:22:01.381 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 147 (ObjNum 310 snap 84 seq_num 310) dirty exists 2024-09-13T14:22:01.381 INFO:tasks.rados.rados.0.smithi026.stdout:923: left oid 47 (ObjNum 310 snap 84 seq_num 310) 2024-09-13T14:22:01.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: mgrmap e31: y(active, since 1.01607s) 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[39182]: pgmap v4: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: mgrmap e31: y(active, since 1.01607s) 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-mon[27770]: pgmap v4: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:01.504 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:22:01.505 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: mgrmap e31: y(active, since 1.01607s) 2024-09-13T14:22:01.505 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:01.505 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:01 smithi145 ceph-mon[31057]: pgmap v4: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.915 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:01.655+0000 7fc21f4a7640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:22:01] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: Traceback (most recent call last): 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._connections.run(self.expiration_interval) 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._run(expiration_interval) 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:01.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: s = self.context.wrap_socket( 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: return self.sslsocket_class._create( 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self.do_handshake() 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._sslobj.do_handshake() 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:01.917 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:921: done (4 left) 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:922: done (3 left) 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:923: done (2 left) 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:924: done (1 left) 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:925: done (0 left) 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 85 2024-09-13T14:22:02.369 INFO:tasks.rados.rados.0.smithi026.stdout:926: seq_num 311 ranges {0=417792} 2024-09-13T14:22:02.371 INFO:tasks.rados.rados.0.smithi026.stdout:926: writing smithi02683976-15 from 0 to 417792 tid 1 2024-09-13T14:22:02.371 INFO:tasks.rados.rados.0.smithi026.stdout:927: delete oid 31 current snap is 85 2024-09-13T14:22:02.375 INFO:tasks.rados.rados.0.smithi026.stdout:926: finishing write tid 1 to smithi02683976-15 2024-09-13T14:22:02.375 INFO:tasks.rados.rados.0.smithi026.stdout:926: finishing write tid 2 to smithi02683976-15 2024-09-13T14:22:02.376 INFO:tasks.rados.rados.0.smithi026.stdout:927: done (1 left) 2024-09-13T14:22:02.377 INFO:tasks.rados.rados.0.smithi026.stdout:928: snap_create 2024-09-13T14:22:02.380 INFO:tasks.rados.rados.0.smithi026.stdout:926: finishing write tid 3 to smithi02683976-15 2024-09-13T14:22:02.380 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 192 (ObjNum 311 snap 85 seq_num 311) dirty exists 2024-09-13T14:22:02.380 INFO:tasks.rados.rados.0.smithi026.stdout:926: left oid 15 (ObjNum 311 snap 85 seq_num 311) 2024-09-13T14:22:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:22:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: mgrmap e32: y(active, since 2s) 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: [13/Sep/2024:14:22:01] ENGINE Bus STARTING 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: [13/Sep/2024:14:22:01] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: [13/Sep/2024:14:22:01] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: [13/Sep/2024:14:22:01] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[27770]: [13/Sep/2024:14:22:01] ENGINE Bus STARTED 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:22:02.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: mgrmap e32: y(active, since 2s) 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:01] ENGINE Bus STARTING 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:01] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:01] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:02.418 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.do_handshake() 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:01] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:02.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:02 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:01] ENGINE Bus STARTED 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: mgrmap e32: y(active, since 2s) 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:01] ENGINE Bus STARTING 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:01] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:01] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout: Traceback (most recent call last): 2024-09-13T14:22:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._run(expiration_interval) 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:02.456 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: self.do_handshake() 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:01] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:02.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:02 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:01] ENGINE Bus STARTED 2024-09-13T14:22:03.376 INFO:tasks.rados.rados.0.smithi026.stdout:926: done (1 left) 2024-09-13T14:22:03.376 INFO:tasks.rados.rados.0.smithi026.stdout:928: done (0 left) 2024-09-13T14:22:03.376 INFO:tasks.rados.rados.0.smithi026.stdout:929: copy_from oid 33 from oid 44 current snap is 86 2024-09-13T14:22:03.376 INFO:tasks.rados.rados.0.smithi026.stdout:930: delete oid 3 current snap is 86 2024-09-13T14:22:03.379 INFO:tasks.rados.rados.0.smithi026.stdout:930: done (1 left) 2024-09-13T14:22:03.379 INFO:tasks.rados.rados.0.smithi026.stdout:931: snap_remove snap 80 2024-09-13T14:22:03.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:03 smithi026 ceph-mon[27770]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:22:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:03 smithi026 ceph-mon[39182]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:22:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:03 smithi145 ceph-mon[31057]: osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:22:03.758 INFO:tasks.rados.rados.0.smithi026.stdout:931: done (1 left) 2024-09-13T14:22:03.758 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 86 2024-09-13T14:22:03.758 INFO:tasks.rados.rados.0.smithi026.stdout:929: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:22:03.758 INFO:tasks.rados.rados.0.smithi026.stdout:932: seq_num 314 ranges {2121728=466944} 2024-09-13T14:22:03.772 INFO:tasks.rados.rados.0.smithi026.stdout:932: writing smithi02683976-21 from 2121728 to 2588672 tid 1 2024-09-13T14:22:03.772 INFO:tasks.rados.rados.0.smithi026.stdout:929: finishing copy_from to smithi02683976-33 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 242 (ObjNum 244 snap 61 seq_num 244) dirty exists 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:933: read oid 3 snap -1 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:933: expect deleted 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:929: done (2 left) 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:934: read oid 39 snap -1 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:934: expect deleted 2024-09-13T14:22:03.773 INFO:tasks.rados.rados.0.smithi026.stdout:935: delete oid 37 current snap is 86 2024-09-13T14:22:03.780 INFO:tasks.rados.rados.0.smithi026.stdout:932: finishing write tid 1 to smithi02683976-21 2024-09-13T14:22:03.780 INFO:tasks.rados.rados.0.smithi026.stdout:932: finishing write tid 2 to smithi02683976-21 2024-09-13T14:22:03.781 INFO:tasks.rados.rados.0.smithi026.stdout:933: done (3 left) 2024-09-13T14:22:03.781 INFO:tasks.rados.rados.0.smithi026.stdout:934: done (2 left) 2024-09-13T14:22:03.781 INFO:tasks.rados.rados.0.smithi026.stdout:935: done (1 left) 2024-09-13T14:22:03.781 INFO:tasks.rados.rados.0.smithi026.stdout:936: read oid 14 snap -1 2024-09-13T14:22:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:936: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:22:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:937: delete oid 41 current snap is 86 2024-09-13T14:22:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:932: finishing write tid 3 to smithi02683976-21 2024-09-13T14:22:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 497 (ObjNum 314 snap 86 seq_num 314) dirty exists 2024-09-13T14:22:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:932: left oid 21 (ObjNum 314 snap 86 seq_num 314) 2024-09-13T14:22:03.785 INFO:tasks.rados.rados.0.smithi026.stdout:932: done (2 left) 2024-09-13T14:22:03.785 INFO:tasks.rados.rados.0.smithi026.stdout:937: done (1 left) 2024-09-13T14:22:03.785 INFO:tasks.rados.rados.0.smithi026.stdout:938: read oid 48 snap -1 2024-09-13T14:22:03.785 INFO:tasks.rados.rados.0.smithi026.stdout:938: expect (ObjNum 280 snap 71 seq_num 280) 2024-09-13T14:22:03.786 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 86 2024-09-13T14:22:03.786 INFO:tasks.rados.rados.0.smithi026.stdout:939: seq_num 317 ranges {114688=516096} 2024-09-13T14:22:03.789 INFO:tasks.rados.rados.0.smithi026.stdout:939: writing smithi02683976-17 from 114688 to 630784 tid 1 2024-09-13T14:22:03.789 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 86 2024-09-13T14:22:03.789 INFO:tasks.rados.rados.0.smithi026.stdout:940: seq_num 318 ranges {778240=532480} 2024-09-13T14:22:03.797 INFO:tasks.rados.rados.0.smithi026.stdout:940: writing smithi02683976-8 from 778240 to 1310720 tid 1 2024-09-13T14:22:03.797 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 86 2024-09-13T14:22:03.797 INFO:tasks.rados.rados.0.smithi026.stdout:941: seq_num 319 ranges {0=548864} 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:941: writing smithi02683976-32 from 0 to 548864 tid 1 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:942: rmattr oid 38 current snap is 86 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:939: finishing write tid 1 to smithi02683976-17 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:939: finishing write tid 2 to smithi02683976-17 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:939: finishing write tid 3 to smithi02683976-17 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 382 (ObjNum 317 snap 86 seq_num 317) dirty exists 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:939: left oid 17 (ObjNum 317 snap 86 seq_num 317) 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:939: done (5 left) 2024-09-13T14:22:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:943: delete oid 3 current snap is 86 2024-09-13T14:22:03.803 INFO:tasks.rados.rados.0.smithi026.stdout:940: finishing write tid 1 to smithi02683976-8 2024-09-13T14:22:03.803 INFO:tasks.rados.rados.0.smithi026.stdout:940: finishing write tid 2 to smithi02683976-8 2024-09-13T14:22:03.804 INFO:tasks.rados.rados.0.smithi026.stdout:940: finishing write tid 3 to smithi02683976-8 2024-09-13T14:22:03.804 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 206 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:03.805 INFO:tasks.rados.rados.0.smithi026.stdout:940: left oid 8 (ObjNum 318 snap 86 seq_num 318) 2024-09-13T14:22:03.808 INFO:tasks.rados.rados.0.smithi026.stdout:936: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:22:04.146 INFO:tasks.rados.rados.0.smithi026.stdout:938: expect (ObjNum 280 snap 71 seq_num 280) 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:941: finishing write tid 1 to smithi02683976-32 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 0 (ObjNum 214 snap 53 seq_num 214) dirty exists 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:941: finishing write tid 2 to smithi02683976-32 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:941: finishing write tid 3 to smithi02683976-32 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 388 (ObjNum 319 snap 86 seq_num 319) dirty exists 2024-09-13T14:22:04.293 INFO:tasks.rados.rados.0.smithi026.stdout:941: left oid 32 (ObjNum 319 snap 86 seq_num 319) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:936: done (5 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:938: done (4 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:940: done (3 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:941: done (2 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:942: done (1 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:943: done (0 left) 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:944: copy_from oid 7 from oid 43 current snap is 86 2024-09-13T14:22:04.294 INFO:tasks.rados.rados.0.smithi026.stdout:945: snap_create 2024-09-13T14:22:04.301 INFO:tasks.rados.rados.0.smithi026.stdout:944: finishing copy_from racing read to smithi02683976-7 2024-09-13T14:22:04.302 INFO:tasks.rados.rados.0.smithi026.stdout:944: finishing copy_from to smithi02683976-7 2024-09-13T14:22:04.302 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 496 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:22:04.639 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-mon[31057]: pgmap v8: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-mon[31057]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-mon[31057]: mgrmap e33: y(active, since 4s) 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-mon[31057]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:22:04] ENGINE Bus STARTING 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: CherryPy Checker: 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: The Application mounted at '' has an empty config. 2024-09-13T14:22:04.640 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: 2024-09-13T14:22:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[27770]: pgmap v8: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[27770]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:22:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[27770]: mgrmap e33: y(active, since 4s) 2024-09-13T14:22:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[27770]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:22:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[39182]: pgmap v8: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[39182]: osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:22:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[39182]: mgrmap e33: y(active, since 4s) 2024-09-13T14:22:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:04 smithi026 ceph-mon[39182]: osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:22:04.760 INFO:tasks.rados.rados.0.smithi026.stdout:944: done (1 left) 2024-09-13T14:22:04.760 INFO:tasks.rados.rados.0.smithi026.stdout:945: done (0 left) 2024-09-13T14:22:04.760 INFO:tasks.rados.rados.0.smithi026.stdout:946: rollback oid 44 current snap is 87 2024-09-13T14:22:04.760 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 74 2024-09-13T14:22:04.760 INFO:tasks.rados.rados.0.smithi026.stdout:947: copy_from oid 18 from oid 38 current snap is 87 2024-09-13T14:22:04.761 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 87 2024-09-13T14:22:04.761 INFO:tasks.rados.rados.0.smithi026.stdout:948: seq_num 321 ranges {0=581632} 2024-09-13T14:22:04.763 INFO:tasks.rados.rados.0.smithi026.stdout:948: writing smithi02683976-3 from 0 to 581632 tid 1 2024-09-13T14:22:04.763 INFO:tasks.rados.rados.0.smithi026.stdout:949: snap_remove snap 82 2024-09-13T14:22:04.899 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:22:04] ENGINE Serving on http://:::9283 2024-09-13T14:22:04.899 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: [13/Sep/2024:14:22:04] ENGINE Bus STARTED 2024-09-13T14:22:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:22:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:05 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.710 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: Standby manager daemon x started 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2961613735' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.711 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.712 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:05 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:05.762 INFO:tasks.rados.rados.0.smithi026.stdout:949: done (3 left) 2024-09-13T14:22:05.762 INFO:tasks.rados.rados.0.smithi026.stdout:950: snap_create 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:946: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:946: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:946: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 419 (ObjNum 244 snap 61 seq_num 244) dirty exists 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:948: finishing write tid 1 to smithi02683976-3 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:948: finishing write tid 2 to smithi02683976-3 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:948: finishing write tid 3 to smithi02683976-3 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 475 (ObjNum 321 snap 87 seq_num 321) dirty exists 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:948: left oid 3 (ObjNum 321 snap 87 seq_num 321) 2024-09-13T14:22:05.763 INFO:tasks.rados.rados.0.smithi026.stdout:947: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:22:05.764 INFO:tasks.rados.rados.0.smithi026.stdout:947: finishing copy_from to smithi02683976-18 2024-09-13T14:22:05.764 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 282 (ObjNum 214 snap 53 seq_num 214) dirty exists 2024-09-13T14:22:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: pgmap v12: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: pgmap v12: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:06 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:946: done (3 left) 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:947: done (2 left) 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:948: done (1 left) 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:950: done (0 left) 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 88 2024-09-13T14:22:06.765 INFO:tasks.rados.rados.0.smithi026.stdout:951: seq_num 322 ranges {0=598016} 2024-09-13T14:22:06.768 INFO:tasks.rados.rados.0.smithi026.stdout:951: writing smithi02683976-1 from 0 to 598016 tid 1 2024-09-13T14:22:06.768 INFO:tasks.rados.rados.0.smithi026.stdout:952: read oid 35 snap -1 2024-09-13T14:22:06.768 INFO:tasks.rados.rados.0.smithi026.stdout:952: expect (ObjNum 275 snap 70 seq_num 275) 2024-09-13T14:22:06.768 INFO:tasks.rados.rados.0.smithi026.stdout:953: snap_create 2024-09-13T14:22:06.774 INFO:tasks.rados.rados.0.smithi026.stdout:951: finishing write tid 1 to smithi02683976-1 2024-09-13T14:22:06.774 INFO:tasks.rados.rados.0.smithi026.stdout:951: finishing write tid 2 to smithi02683976-1 2024-09-13T14:22:06.777 INFO:tasks.rados.rados.0.smithi026.stdout:951: finishing write tid 3 to smithi02683976-1 2024-09-13T14:22:06.777 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 479 (ObjNum 322 snap 88 seq_num 322) dirty exists 2024-09-13T14:22:06.777 INFO:tasks.rados.rados.0.smithi026.stdout:951: left oid 1 (ObjNum 322 snap 88 seq_num 322) 2024-09-13T14:22:06.783 INFO:tasks.rados.rados.0.smithi026.stdout:952: expect (ObjNum 275 snap 70 seq_num 275) 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: pgmap v12: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:06.925 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:07.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:07.769 INFO:tasks.rados.rados.0.smithi026.stdout:951: done (2 left) 2024-09-13T14:22:07.796 INFO:tasks.rados.rados.0.smithi026.stdout:952: done (1 left) 2024-09-13T14:22:07.796 INFO:tasks.rados.rados.0.smithi026.stdout:953: done (0 left) 2024-09-13T14:22:07.796 INFO:tasks.rados.rados.0.smithi026.stdout:954: rollback oid 37 current snap is 89 2024-09-13T14:22:07.796 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 78 2024-09-13T14:22:07.796 INFO:tasks.rados.rados.0.smithi026.stdout:955: delete oid 44 current snap is 89 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:955: done (1 left) 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:956: copy_from oid 39 from oid 8 current snap is 89 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:957: read oid 46 snap 88 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:957: expect deleted 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:958: rollback oid 29 current snap is 89 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 74 2024-09-13T14:22:07.797 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 89 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:959: seq_num 324 ranges {245760=630784} 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:959: writing smithi02683976-50 from 245760 to 876544 tid 1 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 89 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:960: seq_num 325 ranges {2088960=647168} 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:960: writing smithi02683976-24 from 2088960 to 2736128 tid 1 2024-09-13T14:22:07.798 INFO:tasks.rados.rados.0.smithi026.stdout:961: snap_remove snap 79 2024-09-13T14:22:08.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:07 smithi026 ceph-mon[39182]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:22:08.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:07 smithi026 ceph-mon[27770]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:22:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:07 smithi145 ceph-mon[31057]: osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:22:08.760 INFO:tasks.rados.rados.0.smithi026.stdout:961: done (6 left) 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:962: rmattr oid 28 current snap is 89 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:962: done (6 left) 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:963: setattr oid 44 current snap is 89 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:954: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:954: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 290 (ObjNum 288 snap 78 seq_num 288) dirty exists 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:958: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:958: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:22:08.761 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 509 (ObjNum 33 snap 0 seq_num 518853888) dirty dne 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:956: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:956: finishing copy_from to smithi02683976-39 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 206 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:959: finishing write tid 1 to smithi02683976-50 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:959: finishing write tid 2 to smithi02683976-50 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:959: finishing write tid 3 to smithi02683976-50 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 508 (ObjNum 324 snap 89 seq_num 324) dirty exists 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:959: left oid 50 (ObjNum 324 snap 89 seq_num 324) 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:960: finishing write tid 1 to smithi02683976-24 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:960: finishing write tid 2 to smithi02683976-24 2024-09-13T14:22:08.762 INFO:tasks.rados.rados.0.smithi026.stdout:960: finishing write tid 3 to smithi02683976-24 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 267 (ObjNum 325 snap 89 seq_num 325) dirty exists 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:960: left oid 24 (ObjNum 325 snap 89 seq_num 325) 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:954: done (6 left) 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:956: done (5 left) 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:957: done (4 left) 2024-09-13T14:22:08.763 INFO:tasks.rados.rados.0.smithi026.stdout:958: done (3 left) 2024-09-13T14:22:08.764 INFO:tasks.rados.rados.0.smithi026.stdout:959: done (2 left) 2024-09-13T14:22:08.764 INFO:tasks.rados.rados.0.smithi026.stdout:960: done (1 left) 2024-09-13T14:22:08.764 INFO:tasks.rados.rados.0.smithi026.stdout:964: snap_remove snap 83 2024-09-13T14:22:08.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: pgmap v15: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[39182]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: pgmap v15: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:22:08.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:08.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:08.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:08.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:08.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:08 smithi026 ceph-mon[27770]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: pgmap v15: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:08 smithi145 ceph-mon[31057]: osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:22:09.761 INFO:tasks.rados.rados.0.smithi026.stdout:964: done (1 left) 2024-09-13T14:22:09.761 INFO:tasks.rados.rados.0.smithi026.stdout:965: rollback oid 16 current snap is 89 2024-09-13T14:22:09.761 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 74 2024-09-13T14:22:09.761 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 421 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:22:09.761 INFO:tasks.rados.rados.0.smithi026.stdout:963: done (1 left) 2024-09-13T14:22:09.762 INFO:tasks.rados.rados.0.smithi026.stdout:966: snap_remove snap 84 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[27770]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:09 smithi026 ceph-mon[39182]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[66778]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:22:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:10.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:09 smithi145 ceph-mon[31057]: osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:22:10.501 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:10.762 INFO:tasks.rados.rados.0.smithi026.stdout:966: done (1 left) 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:967: read oid 46 snap -1 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:967: expect deleted 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:968: copy_from oid 21 from oid 11 current snap is 89 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:965: finishing rollback tid 0 to smithi02683976-16 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:965: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:965: finishing rollback tid 2 to smithi02683976-16 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 513 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:965: done (2 left) 2024-09-13T14:22:10.763 INFO:tasks.rados.rados.0.smithi026.stdout:969: snap_remove snap 74 2024-09-13T14:22:11.162 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.162 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.162 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.162 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: pgmap v18: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.163 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.164 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:10 smithi145 ceph-mon[31057]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:22:11.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: pgmap v18: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:11.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: pgmap v18: 121 pgs: 121 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: Reconfiguring daemon prometheus.a on smithi145 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[27770]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:11.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:11.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:11.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:11.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:10 smithi026 ceph-mon[39182]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:22:11.765 INFO:tasks.rados.rados.0.smithi026.stdout:969: done (2 left) 2024-09-13T14:22:11.766 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 89 2024-09-13T14:22:11.766 INFO:tasks.rados.rados.0.smithi026.stdout:970: seq_num 326 ranges {630784=663552} 2024-09-13T14:22:11.776 INFO:tasks.rados.rados.0.smithi026.stdout:970: writing smithi02683976-17 from 630784 to 1294336 tid 1 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:971: delete oid 29 current snap is 89 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:968: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:968: finishing copy_from to smithi02683976-21 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 459 (ObjNum 268 snap 69 seq_num 268) dirty exists 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:967: done (3 left) 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:968: done (2 left) 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:971: done (1 left) 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 89 2024-09-13T14:22:11.777 INFO:tasks.rados.rados.0.smithi026.stdout:972: seq_num 328 ranges {8192=696320} 2024-09-13T14:22:11.779 INFO:tasks.rados.rados.0.smithi026.stdout:972: writing smithi02683976-2 from 8192 to 704512 tid 1 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:973: rollback oid 36 current snap is 89 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 86 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:970: finishing write tid 1 to smithi02683976-17 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:970: finishing write tid 2 to smithi02683976-17 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:974: snap_create 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:970: finishing write tid 3 to smithi02683976-17 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 384 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:22:11.780 INFO:tasks.rados.rados.0.smithi026.stdout:970: left oid 17 (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:22:11.787 INFO:tasks.rados.rados.0.smithi026.stdout:972: finishing write tid 1 to smithi02683976-2 2024-09-13T14:22:11.787 INFO:tasks.rados.rados.0.smithi026.stdout:972: finishing write tid 2 to smithi02683976-2 2024-09-13T14:22:11.789 INFO:tasks.rados.rados.0.smithi026.stdout:973: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:22:11.789 INFO:tasks.rados.rados.0.smithi026.stdout:973: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:22:11.790 INFO:tasks.rados.rados.0.smithi026.stdout:972: finishing write tid 3 to smithi02683976-2 2024-09-13T14:22:11.790 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 282 (ObjNum 328 snap 89 seq_num 328) dirty exists 2024-09-13T14:22:11.790 INFO:tasks.rados.rados.0.smithi026.stdout:972: left oid 2 (ObjNum 328 snap 89 seq_num 328) 2024-09-13T14:22:11.791 INFO:tasks.rados.rados.0.smithi026.stdout:973: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:22:11.791 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 487 (ObjNum 165 snap 38 seq_num 165) dirty exists 2024-09-13T14:22:12.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:11 smithi145 ceph-mon[31057]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-13T14:22:12.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:11 smithi145 ceph-mon[31057]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:22:12.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:11 smithi026 ceph-mon[27770]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-13T14:22:12.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:11 smithi026 ceph-mon[27770]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:22:12.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:11 smithi026 ceph-mon[39182]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-13T14:22:12.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:11 smithi026 ceph-mon[39182]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:970: done (3 left) 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:972: done (2 left) 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:973: done (1 left) 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:974: done (0 left) 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:975: rmattr oid 32 current snap is 90 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:975: done (0 left) 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:976: read oid 50 snap -1 2024-09-13T14:22:12.787 INFO:tasks.rados.rados.0.smithi026.stdout:976: expect (ObjNum 324 snap 89 seq_num 324) 2024-09-13T14:22:12.788 INFO:tasks.rados.rados.0.smithi026.stdout:977: rollback oid 48 current snap is 90 2024-09-13T14:22:12.788 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 76 2024-09-13T14:22:12.788 INFO:tasks.rados.rados.0.smithi026.stdout:978: delete oid 5 current snap is 90 2024-09-13T14:22:12.796 INFO:tasks.rados.rados.0.smithi026.stdout:976: expect (ObjNum 324 snap 89 seq_num 324) 2024-09-13T14:22:12.821 INFO:tasks.rados.rados.0.smithi026.stdout:977: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:22:12.821 INFO:tasks.rados.rados.0.smithi026.stdout:977: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:22:12.821 INFO:tasks.rados.rados.0.smithi026.stdout:977: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:22:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 155 (ObjNum 280 snap 71 seq_num 280) dirty exists 2024-09-13T14:22:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:976: done (2 left) 2024-09-13T14:22:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:977: done (1 left) 2024-09-13T14:22:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:978: done (0 left) 2024-09-13T14:22:12.822 INFO:tasks.rados.rados.0.smithi026.stdout:979: delete oid 45 current snap is 90 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:979: done (0 left) 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:980: copy_from oid 25 from oid 31 current snap is 90 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:981: read oid 49 snap 89 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:981: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 90 2024-09-13T14:22:12.825 INFO:tasks.rados.rados.0.smithi026.stdout:982: seq_num 331 ranges {2170880=753664} 2024-09-13T14:22:12.842 INFO:tasks.rados.rados.0.smithi026.stdout:982: writing smithi02683976-23 from 2170880 to 2924544 tid 1 2024-09-13T14:22:12.842 INFO:tasks.rados.rados.0.smithi026.stdout:983: copy_from oid 32 from oid 42 current snap is 90 2024-09-13T14:22:12.842 INFO:tasks.rados.rados.0.smithi026.stdout:980: finishing copy_from to smithi02683976-25 2024-09-13T14:22:12.843 INFO:tasks.rados.rados.0.smithi026.stdout:980: got expected ENOENT (src dne) 2024-09-13T14:22:12.843 INFO:tasks.rados.rados.0.smithi026.stdout:980: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:22:12.843 INFO:tasks.rados.rados.0.smithi026.stdout:981: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:22:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:12 smithi145 ceph-mon[31057]: pgmap v21: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:12 smithi145 ceph-mon[31057]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:982: finishing write tid 1 to smithi02683976-23 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:982: finishing write tid 2 to smithi02683976-23 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:982: finishing write tid 3 to smithi02683976-23 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 400 (ObjNum 331 snap 90 seq_num 331) dirty exists 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:982: left oid 23 (ObjNum 331 snap 90 seq_num 331) 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:983: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:22:12.980 INFO:tasks.rados.rados.0.smithi026.stdout:983: finishing copy_from to smithi02683976-32 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 287 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:980: done (3 left) 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:981: done (2 left) 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:982: done (1 left) 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:983: done (0 left) 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 90 2024-09-13T14:22:12.981 INFO:tasks.rados.rados.0.smithi026.stdout:984: seq_num 332 ranges {876544=770048} 2024-09-13T14:22:12.989 INFO:tasks.rados.rados.0.smithi026.stdout:984: writing smithi02683976-50 from 876544 to 1646592 tid 1 2024-09-13T14:22:12.989 INFO:tasks.rados.rados.0.smithi026.stdout:985: read oid 19 snap 87 2024-09-13T14:22:12.989 INFO:tasks.rados.rados.0.smithi026.stdout:985: expect (ObjNum 255 snap 66 seq_num 255) 2024-09-13T14:22:12.990 INFO:tasks.rados.rados.0.smithi026.stdout:986: read oid 41 snap -1 2024-09-13T14:22:12.990 INFO:tasks.rados.rados.0.smithi026.stdout:986: expect deleted 2024-09-13T14:22:12.990 INFO:tasks.rados.rados.0.smithi026.stdout:987: read oid 7 snap 85 2024-09-13T14:22:12.990 INFO:tasks.rados.rados.0.smithi026.stdout:987: expect deleted 2024-09-13T14:22:12.990 INFO:tasks.rados.rados.0.smithi026.stdout:988: snap_create 2024-09-13T14:22:12.993 INFO:tasks.rados.rados.0.smithi026.stdout:984: finishing write tid 1 to smithi02683976-50 2024-09-13T14:22:12.993 INFO:tasks.rados.rados.0.smithi026.stdout:984: finishing write tid 2 to smithi02683976-50 2024-09-13T14:22:12.994 INFO:tasks.rados.rados.0.smithi026.stdout:984: finishing write tid 3 to smithi02683976-50 2024-09-13T14:22:12.995 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 537 (ObjNum 332 snap 90 seq_num 332) dirty exists 2024-09-13T14:22:12.995 INFO:tasks.rados.rados.0.smithi026.stdout:984: left oid 50 (ObjNum 332 snap 90 seq_num 332) 2024-09-13T14:22:13.018 INFO:tasks.rados.rados.0.smithi026.stdout:985: expect (ObjNum 255 snap 66 seq_num 255) 2024-09-13T14:22:13.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:12 smithi026 ceph-mon[27770]: pgmap v21: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:13.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:12 smithi026 ceph-mon[27770]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:22:13.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:12 smithi026 ceph-mon[39182]: pgmap v21: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:13.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:12 smithi026 ceph-mon[39182]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:22:13.370 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 systemd[1]: Stopping Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T14:22:13.669 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.369Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.370Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.371Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.371Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[76246]: ts=2024-09-13T14:22:13.371Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T14:22:13.670 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 podman[79883]: 2024-09-13 14:22:13.380099018 +0000 UTC m=+0.107492697 container died 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:13.761 INFO:tasks.rados.rados.0.smithi026.stdout:984: done (4 left) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:985: done (3 left) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:986: done (2 left) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:987: done (1 left) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:988: done (0 left) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:989: read oid 3 snap -1 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:989: expect (ObjNum 321 snap 87 seq_num 321) 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:990: copy_from oid 50 from oid 8 current snap is 91 2024-09-13T14:22:13.762 INFO:tasks.rados.rados.0.smithi026.stdout:991: snap_remove snap 89 2024-09-13T14:22:13.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 podman[79883]: 2024-09-13 14:22:13.776035224 +0000 UTC m=+0.503428903 container cleanup 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:13.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:13 smithi145 bash[79883]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:22:14.358 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 podman[79901]: 2024-09-13 14:22:14.018855555 +0000 UTC m=+0.639799641 container remove 2ee9d61eef4d65a180d55568024d1e3354fc023333ee492ad2737574d7c54f24 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:14.359 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:22:14.359 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 systemd[1]: Stopped Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:14.359 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.144s CPU time. 2024-09-13T14:22:14.359 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 systemd[1]: Starting Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:14.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 podman[79998]: 2024-09-13 14:22:14.262082221 +0000 UTC m=+0.017505212 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:22:14.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 podman[79998]: 2024-09-13 14:22:14.375545286 +0000 UTC m=+0.130968261 container create 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:14.763 INFO:tasks.rados.rados.0.smithi026.stdout:991: done (2 left) 2024-09-13T14:22:14.763 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 91 2024-09-13T14:22:14.764 INFO:tasks.rados.rados.0.smithi026.stdout:992: seq_num 333 ranges {2334720=786432} 2024-09-13T14:22:14.781 INFO:tasks.rados.rados.0.smithi026.stdout:992: writing smithi02683976-38 from 2334720 to 3121152 tid 1 2024-09-13T14:22:14.781 INFO:tasks.rados.rados.0.smithi026.stdout:993: read oid 20 snap -1 2024-09-13T14:22:14.781 INFO:tasks.rados.rados.0.smithi026.stdout:993: expect deleted 2024-09-13T14:22:14.781 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 91 2024-09-13T14:22:14.781 INFO:tasks.rados.rados.0.smithi026.stdout:994: seq_num 334 ranges {0=802816} 2024-09-13T14:22:14.786 INFO:tasks.rados.rados.0.smithi026.stdout:994: writing smithi02683976-41 from 0 to 802816 tid 1 2024-09-13T14:22:14.787 INFO:tasks.rados.rados.0.smithi026.stdout:995: delete oid 11 current snap is 91 2024-09-13T14:22:14.787 INFO:tasks.rados.rados.0.smithi026.stdout:989: expect (ObjNum 321 snap 87 seq_num 321) 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:990: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:990: finishing copy_from to smithi02683976-50 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 206 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:989: done (5 left) 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:990: done (4 left) 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:995: done (3 left) 2024-09-13T14:22:14.806 INFO:tasks.rados.rados.0.smithi026.stdout:996: snap_remove snap 86 2024-09-13T14:22:15.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:14 smithi026 ceph-mon[27770]: pgmap v24: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:15.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:14 smithi026 ceph-mon[27770]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:22:15.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:14 smithi026 ceph-mon[39182]: pgmap v24: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:15.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:14 smithi026 ceph-mon[39182]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:22:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-mon[31057]: pgmap v24: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 778 KiB/s wr, 10 op/s 2024-09-13T14:22:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-mon[31057]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:22:15.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 podman[79998]: 2024-09-13 14:22:14.864106642 +0000 UTC m=+0.619529618 container init 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 podman[79998]: 2024-09-13 14:22:14.868711967 +0000 UTC m=+0.624134942 container start 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.887Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.887Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.887Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi145 (none))" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.887Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.887Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.889Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.889Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.891Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.891Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.893Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.893Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.902µs 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.893Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.893Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.905Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.910Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.910Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.910Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.983µs wal_replay_duration=17.2121ms wbl_replay_duration=101ns total_replay_duration=17.272038ms 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.914Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.914Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.914Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.930Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.256707ms db_storage=954ns remote_storage=1.151µs web_handler=386ns query_engine=685ns scrape=690.577µs scrape_sd=206.244µs notify=25.121µs notify_sd=21.057µs rules=14.9384ms tracing=6.777µs 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.930Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:14.930Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 bash[79998]: 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 2024-09-13T14:22:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:14 smithi145 systemd[1]: Started Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:15.770 INFO:tasks.rados.rados.0.smithi026.stdout:996: done (3 left) 2024-09-13T14:22:15.770 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 91 2024-09-13T14:22:15.770 INFO:tasks.rados.rados.0.smithi026.stdout:997: seq_num 336 ranges {2736128=434176,3170304=401408} 2024-09-13T14:22:15.789 INFO:tasks.rados.rados.0.smithi026.stdout:997: writing smithi02683976-24 from 2736128 to 3170304 tid 1 2024-09-13T14:22:15.792 INFO:tasks.rados.rados.0.smithi026.stdout:997: writing smithi02683976-24 from 3170304 to 3571712 tid 2 2024-09-13T14:22:15.792 INFO:tasks.rados.rados.0.smithi026.stdout:998: copy_from oid 36 from oid 10 current snap is 91 2024-09-13T14:22:15.792 INFO:tasks.rados.rados.0.smithi026.stdout:992: finishing write tid 1 to smithi02683976-38 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:992: finishing write tid 2 to smithi02683976-38 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:994: finishing write tid 1 to smithi02683976-41 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:994: finishing write tid 2 to smithi02683976-41 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:992: finishing write tid 3 to smithi02683976-38 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 405 (ObjNum 333 snap 91 seq_num 333) dirty exists 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:992: left oid 38 (ObjNum 333 snap 91 seq_num 333) 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:994: finishing write tid 3 to smithi02683976-41 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 386 (ObjNum 334 snap 91 seq_num 334) dirty exists 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:994: left oid 41 (ObjNum 334 snap 91 seq_num 334) 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:992: done (4 left) 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:993: done (3 left) 2024-09-13T14:22:15.793 INFO:tasks.rados.rados.0.smithi026.stdout:994: done (2 left) 2024-09-13T14:22:15.794 INFO:tasks.rados.rados.0.smithi026.stdout:999: snap_remove snap 76 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:16.121 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:15 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:22:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:15 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:16.779 INFO:tasks.rados.rados.0.smithi026.stdout:999: done (2 left) 2024-09-13T14:22:16.780 INFO:tasks.rados.rados.0.smithi026.stdout:1000: snap_remove snap 78 2024-09-13T14:22:17.049 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:17.049 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:17.049 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: pgmap v27: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.050 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:22:17.050 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.050 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.050 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:22:17.050 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:16 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:17.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: pgmap v27: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: pgmap v27: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:22:17.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:22:17.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:22:17.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:16 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:17.786 INFO:tasks.rados.rados.0.smithi026.stdout:1000: done (2 left) 2024-09-13T14:22:17.786 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 91 2024-09-13T14:22:17.786 INFO:tasks.rados.rados.0.smithi026.stdout:997: finishing write tid 1 to smithi02683976-24 2024-09-13T14:22:17.786 INFO:tasks.rados.rados.0.smithi026.stdout:1001: seq_num 337 ranges {0=450560,450560=401408} 2024-09-13T14:22:17.789 INFO:tasks.rados.rados.0.smithi026.stdout:1001: writing smithi02683976-5 from 0 to 450560 tid 1 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:1001: writing smithi02683976-5 from 450560 to 851968 tid 2 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:1002: read oid 42 snap -1 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:1002: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:1003: setattr oid 17 current snap is 91 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:997: finishing write tid 2 to smithi02683976-24 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:997: finishing write tid 3 to smithi02683976-24 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:997: finishing write tid 4 to smithi02683976-24 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 290 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:997: left oid 24 (ObjNum 336 snap 91 seq_num 336) 2024-09-13T14:22:17.793 INFO:tasks.rados.rados.0.smithi026.stdout:998: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:22:17.794 INFO:tasks.rados.rados.0.smithi026.stdout:998: finishing copy_from to smithi02683976-36 2024-09-13T14:22:17.794 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 264 (ObjNum 296 snap 82 seq_num 296) dirty exists 2024-09-13T14:22:17.795 INFO:tasks.rados.rados.0.smithi026.stdout:997: done (4 left) 2024-09-13T14:22:17.795 INFO:tasks.rados.rados.0.smithi026.stdout:998: done (3 left) 2024-09-13T14:22:17.795 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 91 2024-09-13T14:22:17.795 INFO:tasks.rados.rados.0.smithi026.stdout:1004: seq_num 338 ranges {2334720=466944,2801664=401408} 2024-09-13T14:22:17.811 INFO:tasks.rados.rados.0.smithi026.stdout:1004: writing smithi02683976-18 from 2334720 to 2801664 tid 1 2024-09-13T14:22:17.814 INFO:tasks.rados.rados.0.smithi026.stdout:1004: writing smithi02683976-18 from 2801664 to 3203072 tid 2 2024-09-13T14:22:17.814 INFO:tasks.rados.rados.0.smithi026.stdout:1005: delete oid 7 current snap is 91 2024-09-13T14:22:17.814 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 387 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:22:17.815 INFO:tasks.rados.rados.0.smithi026.stdout:1001: finishing write tid 1 to smithi02683976-5 2024-09-13T14:22:17.815 INFO:tasks.rados.rados.0.smithi026.stdout:1002: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:17.856 INFO:tasks.rados.rados.0.smithi026.stdout:1001: finishing write tid 2 to smithi02683976-5 2024-09-13T14:22:17.856 INFO:tasks.rados.rados.0.smithi026.stdout:1001: finishing write tid 3 to smithi02683976-5 2024-09-13T14:22:17.856 INFO:tasks.rados.rados.0.smithi026.stdout:1001: finishing write tid 4 to smithi02683976-5 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 506 (ObjNum 337 snap 91 seq_num 337) dirty exists 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1001: left oid 5 (ObjNum 337 snap 91 seq_num 337) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1001: done (4 left) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1002: done (3 left) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1003: done (2 left) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1005: done (1 left) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1006: read oid 5 snap -1 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1006: expect (ObjNum 337 snap 91 seq_num 337) 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1007: rmattr oid 32 current snap is 91 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1004: finishing write tid 1 to smithi02683976-18 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1004: finishing write tid 2 to smithi02683976-18 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1004: finishing write tid 3 to smithi02683976-18 2024-09-13T14:22:17.857 INFO:tasks.rados.rados.0.smithi026.stdout:1008: rmattr oid 50 current snap is 91 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:1004: finishing write tid 4 to smithi02683976-18 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 435 (ObjNum 338 snap 91 seq_num 338) dirty exists 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:1004: left oid 18 (ObjNum 338 snap 91 seq_num 338) 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:1004: done (3 left) 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:1009: setattr oid 20 current snap is 91 2024-09-13T14:22:17.858 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 566 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:17.859 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 437 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1007: done (3 left) 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1008: done (2 left) 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1010: read oid 39 snap 75 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1010: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1011: rollback oid 45 current snap is 91 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 87 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1012: setattr oid 46 current snap is 91 2024-09-13T14:22:17.860 INFO:tasks.rados.rados.0.smithi026.stdout:1006: expect (ObjNum 337 snap 91 seq_num 337) 2024-09-13T14:22:17.884 INFO:tasks.rados.rados.0.smithi026.stdout:1011: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:22:17.884 INFO:tasks.rados.rados.0.smithi026.stdout:1011: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:22:17.884 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 286 (ObjNum 276 snap 71 seq_num 276) dirty exists 2024-09-13T14:22:17.884 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 533 (ObjNum 737 snap 0 seq_num 1128147923) dirty exists 2024-09-13T14:22:17.884 INFO:tasks.rados.rados.0.smithi026.stdout:1010: expect (ObjNum 165 snap 38 seq_num 165) 2024-09-13T14:22:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:17 smithi145 ceph-mon[31057]: Upgrade: Updating mgr.x 2024-09-13T14:22:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:17 smithi145 ceph-mon[31057]: Deploying daemon mgr.x on smithi145 2024-09-13T14:22:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:17 smithi145 ceph-mon[31057]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1006: done (4 left) 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1009: done (3 left) 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1010: done (2 left) 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1011: done (1 left) 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1013: rollback oid 9 current snap is 91 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 87 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1014: rollback oid 28 current snap is 91 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 87 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1015: read oid 16 snap -1 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1015: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:22:17.967 INFO:tasks.rados.rados.0.smithi026.stdout:1016: read oid 15 snap 75 2024-09-13T14:22:17.968 INFO:tasks.rados.rados.0.smithi026.stdout:1016: expect deleted 2024-09-13T14:22:17.968 INFO:tasks.rados.rados.0.smithi026.stdout:1017: read oid 7 snap -1 2024-09-13T14:22:17.968 INFO:tasks.rados.rados.0.smithi026.stdout:1017: expect deleted 2024-09-13T14:22:17.968 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 91 2024-09-13T14:22:17.968 INFO:tasks.rados.rados.0.smithi026.stdout:1018: seq_num 340 ranges {0=499712,499712=401408} 2024-09-13T14:22:17.970 INFO:tasks.rados.rados.0.smithi026.stdout:1018: writing smithi02683976-31 from 0 to 499712 tid 1 2024-09-13T14:22:17.973 INFO:tasks.rados.rados.0.smithi026.stdout:1018: writing smithi02683976-31 from 499712 to 901120 tid 2 2024-09-13T14:22:17.973 INFO:tasks.rados.rados.0.smithi026.stdout:1019: snap_remove snap 85 2024-09-13T14:22:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[27770]: Upgrade: Updating mgr.x 2024-09-13T14:22:18.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[27770]: Deploying daemon mgr.x on smithi145 2024-09-13T14:22:18.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[27770]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:22:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[39182]: Upgrade: Updating mgr.x 2024-09-13T14:22:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[39182]: Deploying daemon mgr.x on smithi145 2024-09-13T14:22:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:17 smithi026 ceph-mon[39182]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:22:18.528 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:18 smithi145 systemd[1]: Stopping Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:18.764 INFO:tasks.rados.rados.0.smithi026.stdout:1019: done (7 left) 2024-09-13T14:22:18.764 INFO:tasks.rados.rados.0.smithi026.stdout:1020: setattr oid 23 current snap is 91 2024-09-13T14:22:18.765 INFO:tasks.rados.rados.0.smithi026.stdout:1015: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:22:18.771 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 422 (ObjNum 33 snap 0 seq_num 519192208) dirty exists 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1013: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1013: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1014: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1014: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1013: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 570 (ObjNum 291 snap 80 seq_num 291) dirty exists 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1014: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 299 (ObjNum 307 snap 83 seq_num 307) dirty exists 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1018: finishing write tid 1 to smithi02683976-31 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1018: finishing write tid 2 to smithi02683976-31 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1018: finishing write tid 3 to smithi02683976-31 2024-09-13T14:22:18.772 INFO:tasks.rados.rados.0.smithi026.stdout:1018: finishing write tid 4 to smithi02683976-31 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 573 (ObjNum 340 snap 91 seq_num 340) dirty exists 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1018: left oid 31 (ObjNum 340 snap 91 seq_num 340) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1012: done (7 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1013: done (6 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1014: done (5 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1015: done (4 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1016: done (3 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1017: done (2 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1018: done (1 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1021: rmattr oid 43 current snap is 91 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1021: done (1 left) 2024-09-13T14:22:18.773 INFO:tasks.rados.rados.0.smithi026.stdout:1022: snap_remove snap 88 2024-09-13T14:22:18.782 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:18 smithi145 podman[80400]: 2024-09-13 14:22:18.524626765 +0000 UTC m=+0.139428623 container died a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T14:22:19.093 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:18 smithi145 ceph-mon[31057]: pgmap v30: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:19.094 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:18 smithi145 ceph-mon[31057]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:22:19.094 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:18 smithi145 ceph-mon[31057]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:22:19.094 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:18 smithi145 podman[80400]: 2024-09-13 14:22:18.917044335 +0000 UTC m=+0.531846186 container cleanup a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T14:22:19.094 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:18 smithi145 bash[80400]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:22:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[27770]: pgmap v30: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[27770]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:22:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[27770]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:22:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[39182]: pgmap v30: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[39182]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:22:19.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:18 smithi026 ceph-mon[39182]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:22:19.345 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 podman[80418]: 2024-09-13 14:22:19.093407856 +0000 UTC m=+0.567403627 container remove a4ae2b62581905c1912cf5eedb13c5219cd4a8915f4d327a88f5d656773f006e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:22:19.345 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:22:19.345 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:22:19.345 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 systemd[1]: Stopped Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:19.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service: Consumed 18.596s CPU time. 2024-09-13T14:22:19.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 systemd[1]: Starting Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:19.767 INFO:tasks.rados.rados.0.smithi026.stdout:1022: done (1 left) 2024-09-13T14:22:19.767 INFO:tasks.rados.rados.0.smithi026.stdout:1023: rollback oid 1 current snap is 91 2024-09-13T14:22:19.767 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 87 2024-09-13T14:22:19.767 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 439 (ObjNum 331 snap 90 seq_num 331) dirty exists 2024-09-13T14:22:19.767 INFO:tasks.rados.rados.0.smithi026.stdout:1020: done (1 left) 2024-09-13T14:22:19.768 INFO:tasks.rados.rados.0.smithi026.stdout:1024: delete oid 5 current snap is 91 2024-09-13T14:22:19.777 INFO:tasks.rados.rados.0.smithi026.stdout:1023: finishing rollback tid 0 to smithi02683976-1 2024-09-13T14:22:19.777 INFO:tasks.rados.rados.0.smithi026.stdout:1023: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:22:19.777 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 517 (ObjNum 33 snap 0 seq_num 519511696) dirty dne 2024-09-13T14:22:19.779 INFO:tasks.rados.rados.0.smithi026.stdout:1023: done (1 left) 2024-09-13T14:22:19.779 INFO:tasks.rados.rados.0.smithi026.stdout:1024: done (0 left) 2024-09-13T14:22:19.779 INFO:tasks.rados.rados.0.smithi026.stdout:1025: delete oid 16 current snap is 91 2024-09-13T14:22:19.782 INFO:tasks.rados.rados.0.smithi026.stdout:1025: done (0 left) 2024-09-13T14:22:19.782 INFO:tasks.rados.rados.0.smithi026.stdout:1026: snap_remove snap 75 2024-09-13T14:22:20.022 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 podman[80552]: 2024-09-13 14:22:19.651748642 +0000 UTC m=+0.015909408 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:20.022 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:19 smithi145 podman[80552]: 2024-09-13 14:22:19.753175679 +0000 UTC m=+0.117336477 container create 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:22:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:20.455 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:20 smithi145 podman[80552]: 2024-09-13 14:22:20.022552531 +0000 UTC m=+0.386713283 container init 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T14:22:20.456 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:20 smithi145 podman[80552]: 2024-09-13 14:22:20.027751766 +0000 UTC m=+0.391912519 container start 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:22:20.456 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mgr[80590]: -- 172.21.15.145:0/1370714716 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55de0fa090e0 con 0x55de0f9e6800 2024-09-13T14:22:20.456 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:20 smithi145 bash[80552]: 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda 2024-09-13T14:22:20.456 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:20 smithi145 systemd[1]: Started Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:20.789 INFO:tasks.rados.rados.0.smithi026.stdout:1026: done (0 left) 2024-09-13T14:22:20.789 INFO:tasks.rados.rados.0.smithi026.stdout:1027: snap_remove snap 87 2024-09-13T14:22:21.052 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mon[31057]: pgmap v33: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:21.052 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mon[31057]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:22:21.052 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.052 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.053 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:20 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[39182]: pgmap v33: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:21.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[39182]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[27770]: pgmap v33: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[27770]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:20 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:22:21] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-13T14:22:21.795 INFO:tasks.rados.rados.0.smithi026.stdout:1027: done (0 left) 2024-09-13T14:22:21.796 INFO:tasks.rados.rados.0.smithi026.stdout:1028: snap_create 2024-09-13T14:22:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:21 smithi145 ceph-mon[31057]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:22:22.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:21 smithi026 ceph-mon[39182]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:22:22.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:21 smithi026 ceph-mon[27770]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:22:22.796 INFO:tasks.rados.rados.0.smithi026.stdout:1028: done (0 left) 2024-09-13T14:22:22.796 INFO:tasks.rados.rados.0.smithi026.stdout:1029: rollback oid 48 current snap is 92 2024-09-13T14:22:22.796 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 91 2024-09-13T14:22:22.797 INFO:tasks.rados.rados.0.smithi026.stdout:1030: copy_from oid 4 from oid 24 current snap is 92 2024-09-13T14:22:22.797 INFO:tasks.rados.rados.0.smithi026.stdout:1031: snap_create 2024-09-13T14:22:22.805 INFO:tasks.rados.rados.0.smithi026.stdout:1029: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:22:22.806 INFO:tasks.rados.rados.0.smithi026.stdout:1029: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:22:22.806 INFO:tasks.rados.rados.0.smithi026.stdout:1029: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:22:22.806 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 181 (ObjNum 280 snap 71 seq_num 280) dirty exists 2024-09-13T14:22:22.820 INFO:tasks.rados.rados.0.smithi026.stdout:1030: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:22:22.822 INFO:tasks.rados.rados.0.smithi026.stdout:1030: finishing copy_from to smithi02683976-4 2024-09-13T14:22:22.823 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 290 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:22:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:22 smithi145 ceph-mon[31057]: pgmap v36: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:22 smithi145 ceph-mon[31057]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:22:23.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:22 smithi026 ceph-mon[39182]: pgmap v36: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:23.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:22 smithi026 ceph-mon[39182]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:22:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:22 smithi026 ceph-mon[27770]: pgmap v36: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:23.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:22 smithi026 ceph-mon[27770]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:22:23.765 INFO:tasks.rados.rados.0.smithi026.stdout:1029: done (2 left) 2024-09-13T14:22:23.765 INFO:tasks.rados.rados.0.smithi026.stdout:1030: done (1 left) 2024-09-13T14:22:23.765 INFO:tasks.rados.rados.0.smithi026.stdout:1031: done (0 left) 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1032: read oid 34 snap 91 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1032: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1033: read oid 46 snap -1 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1033: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1034: read oid 17 snap -1 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1034: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1035: rmattr oid 50 current snap is 93 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 93 2024-09-13T14:22:23.766 INFO:tasks.rados.rados.0.smithi026.stdout:1036: seq_num 343 ranges {8396800=548864,8945664=401408} 2024-09-13T14:22:23.815 INFO:tasks.rados.rados.0.smithi026.stdout:1036: writing smithi02683976-13 from 8396800 to 8945664 tid 1 2024-09-13T14:22:23.817 INFO:tasks.rados.rados.0.smithi026.stdout:1036: writing smithi02683976-13 from 8945664 to 9347072 tid 2 2024-09-13T14:22:23.818 INFO:tasks.rados.rados.0.smithi026.stdout:1037: delete oid 3 current snap is 93 2024-09-13T14:22:23.818 INFO:tasks.rados.rados.0.smithi026.stdout:1033: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:22:23.824 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 607 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:23.824 INFO:tasks.rados.rados.0.smithi026.stdout:1032: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:22:24.100 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:23 smithi145 ceph-mon[31057]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:22:24.100 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:23 smithi145 ceph-mon[31057]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:22:24.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:23 smithi026 ceph-mon[39182]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:22:24.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:23 smithi026 ceph-mon[39182]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:22:24.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:23 smithi026 ceph-mon[27770]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:22:24.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:23 smithi026 ceph-mon[27770]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:22:24.414 INFO:tasks.rados.rados.0.smithi026.stdout:1034: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:22:24.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:24.463 INFO:tasks.rados.rados.0.smithi026.stdout:1036: finishing write tid 1 to smithi02683976-13 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:1036: finishing write tid 2 to smithi02683976-13 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:1036: finishing write tid 3 to smithi02683976-13 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:1036: finishing write tid 4 to smithi02683976-13 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 553 (ObjNum 343 snap 93 seq_num 343) dirty exists 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:1036: left oid 13 (ObjNum 343 snap 93 seq_num 343) 2024-09-13T14:22:24.464 INFO:tasks.rados.rados.0.smithi026.stdout:1032: done (5 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1033: done (4 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1034: done (3 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1035: done (2 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1036: done (1 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1037: done (0 left) 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1038: delete oid 46 current snap is 93 2024-09-13T14:22:24.465 INFO:tasks.rados.rados.0.smithi026.stdout:1038: done (0 left) 2024-09-13T14:22:24.466 INFO:tasks.rados.rados.0.smithi026.stdout:1039: copy_from oid 23 from oid 4 current snap is 93 2024-09-13T14:22:24.466 INFO:tasks.rados.rados.0.smithi026.stdout:1040: snap_create 2024-09-13T14:22:24.482 INFO:tasks.rados.rados.0.smithi026.stdout:1039: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:22:24.484 INFO:tasks.rados.rados.0.smithi026.stdout:1039: finishing copy_from to smithi02683976-23 2024-09-13T14:22:24.484 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 290 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:22:24.812 INFO:tasks.rados.rados.0.smithi026.stdout:1039: done (1 left) 2024-09-13T14:22:24.812 INFO:tasks.rados.rados.0.smithi026.stdout:1040: done (0 left) 2024-09-13T14:22:24.812 INFO:tasks.rados.rados.0.smithi026.stdout:1041: rollback oid 30 current snap is 94 2024-09-13T14:22:24.812 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 93 2024-09-13T14:22:24.812 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 94 2024-09-13T14:22:24.813 INFO:tasks.rados.rados.0.smithi026.stdout:1042: seq_num 346 ranges {6905856=598016,7503872=401408} 2024-09-13T14:22:24.854 INFO:tasks.rados.rados.0.smithi026.stdout:1042: writing smithi02683976-26 from 6905856 to 7503872 tid 1 2024-09-13T14:22:24.857 INFO:tasks.rados.rados.0.smithi026.stdout:1042: writing smithi02683976-26 from 7503872 to 7905280 tid 2 2024-09-13T14:22:24.857 INFO:tasks.rados.rados.0.smithi026.stdout:1043: snap_remove snap 90 2024-09-13T14:22:25.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[39182]: pgmap v39: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:25.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[27770]: pgmap v39: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:25.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:24 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:24 smithi145 ceph-mon[31057]: pgmap v39: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.9 MiB/s wr, 34 op/s 2024-09-13T14:22:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:24 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:24 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:25] ENGINE Bus STARTING 2024-09-13T14:22:25.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: CherryPy Checker: 2024-09-13T14:22:25.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: The Application mounted at '' has an empty config. 2024-09-13T14:22:25.705 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: 2024-09-13T14:22:25.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:25] ENGINE Serving on http://:::9283 2024-09-13T14:22:25.706 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:25] ENGINE Bus STARTED 2024-09-13T14:22:25.819 INFO:tasks.rados.rados.0.smithi026.stdout:1043: done (2 left) 2024-09-13T14:22:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:1044: read oid 42 snap -1 2024-09-13T14:22:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:1044: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:1045: snap_remove snap 92 2024-09-13T14:22:25.973 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:22:25.973 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.973 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:25.973 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: Standby manager daemon x restarted 2024-09-13T14:22:25.973 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:22:25.974 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:25.974 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:25.974 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:25.974 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:25.974 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:25.974 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:26.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: Standby manager daemon x restarted 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:26.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: Standby manager daemon x restarted 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: Standby manager daemon x started 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:26.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:25 smithi026 ceph-mon[27770]: from='mgr.? 172.21.15.145:0/2286607922' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:26.748 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1045: done (3 left) 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1046: snap_create 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1041: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 0 (ObjNum 33 snap 0 seq_num 519442128) dirty dne 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1042: finishing write tid 1 to smithi02683976-26 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1042: finishing write tid 2 to smithi02683976-26 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1042: finishing write tid 3 to smithi02683976-26 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1042: finishing write tid 4 to smithi02683976-26 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 211 (ObjNum 346 snap 94 seq_num 346) dirty exists 2024-09-13T14:22:26.820 INFO:tasks.rados.rados.0.smithi026.stdout:1042: left oid 26 (ObjNum 346 snap 94 seq_num 346) 2024-09-13T14:22:26.821 INFO:tasks.rados.rados.0.smithi026.stdout:1044: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:26.853 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:26 smithi145 ceph-mon[31057]: pgmap v42: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:26.853 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:26 smithi145 ceph-mon[31057]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:22:26.853 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:26 smithi145 ceph-mon[31057]: mgrmap e36: y(active, since 26s), standbys: x 2024-09-13T14:22:27.163 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[27770]: pgmap v42: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[27770]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[27770]: mgrmap e36: y(active, since 26s), standbys: x 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[39182]: pgmap v42: 121 pgs: 121 active+clean; 172 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[39182]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:22:27.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:26 smithi026 ceph-mon[39182]: mgrmap e36: y(active, since 26s), standbys: x 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (70s) 22s ago 5m 17.5M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (47s) 2s ago 5m 79.9M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (7m) 22s ago 7m 14.0M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (7m) 2s ago 7m 20.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (7s) 2s ago 10m 72.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (36s) 22s ago 12m 493M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:22:27.331 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (12m) 22s ago 12m 56.9M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 4970e40c75eb 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (11m) 2s ago 11m 52.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (10m) 22s ago 10m 47.5M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 671e5f478faf 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (65s) 22s ago 5m 7680k - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (42s) 2s ago 5m 8174k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (10m) 22s ago 10m 138M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (10m) 22s ago 10m 126M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (9m) 22s ago 9m 129M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (9m) 22s ago 9m 81.3M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:22:27.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (8m) 2s ago 8m 158M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:22:27.333 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (8m) 2s ago 8m 140M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:22:27.333 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (8m) 2s ago 8m 91.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:22:27.333 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (7m) 2s ago 7m 112M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:22:27.333 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (13s) 2s ago 6m 37.1M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:27.681 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 13, 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:22:27.682 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:22:27.830 INFO:tasks.rados.rados.0.smithi026.stdout:1041: done (3 left) 2024-09-13T14:22:27.830 INFO:tasks.rados.rados.0.smithi026.stdout:1042: done (2 left) 2024-09-13T14:22:27.830 INFO:tasks.rados.rados.0.smithi026.stdout:1044: done (1 left) 2024-09-13T14:22:27.830 INFO:tasks.rados.rados.0.smithi026.stdout:1046: done (0 left) 2024-09-13T14:22:27.830 INFO:tasks.rados.rados.0.smithi026.stdout:1047: snap_remove snap 94 2024-09-13T14:22:27.926 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:27 smithi145 ceph-mon[31057]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:27.926 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:27 smithi145 ceph-mon[31057]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:22:27.926 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:27 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/1116362771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:22:27.976 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/20 daemons upgraded", 2024-09-13T14:22:27.977 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T14:22:27.977 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:22:27.977 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:22:27.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[39182]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:27.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[39182]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:22:27.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[39182]: from='client.? 172.21.15.26:0/1116362771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:27.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[27770]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:27.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[27770]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:22:27.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:27 smithi026 ceph-mon[27770]: from='client.? 172.21.15.26:0/1116362771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:28.767 INFO:tasks.rados.rados.0.smithi026.stdout:1047: done (0 left) 2024-09-13T14:22:28.767 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 95 2024-09-13T14:22:28.767 INFO:tasks.rados.rados.0.smithi026.stdout:1048: seq_num 347 ranges {1687552=614400,2301952=401408} 2024-09-13T14:22:28.779 INFO:tasks.rados.rados.0.smithi026.stdout:1048: writing smithi02683976-25 from 1687552 to 2301952 tid 1 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:1048: writing smithi02683976-25 from 2301952 to 2703360 tid 2 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:1049: copy_from oid 1 from oid 10 current snap is 95 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:1050: read oid 46 snap -1 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:1050: expect deleted 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 95 2024-09-13T14:22:28.782 INFO:tasks.rados.rados.0.smithi026.stdout:1051: seq_num 348 ranges {3571712=630784,4202496=401408} 2024-09-13T14:22:28.806 INFO:tasks.rados.rados.0.smithi026.stdout:1051: writing smithi02683976-24 from 3571712 to 4202496 tid 1 2024-09-13T14:22:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:1051: writing smithi02683976-24 from 4202496 to 4603904 tid 2 2024-09-13T14:22:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 95 2024-09-13T14:22:28.809 INFO:tasks.rados.rados.0.smithi026.stdout:1052: seq_num 349 ranges {0=655360,655360=401408} 2024-09-13T14:22:28.813 INFO:tasks.rados.rados.0.smithi026.stdout:1052: writing smithi02683976-5 from 0 to 655360 tid 1 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1052: writing smithi02683976-5 from 655360 to 1056768 tid 2 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1053: rollback oid 26 current snap is 95 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 91 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1048: finishing write tid 1 to smithi02683976-25 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1048: finishing write tid 2 to smithi02683976-25 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1048: finishing write tid 3 to smithi02683976-25 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1048: finishing write tid 4 to smithi02683976-25 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 322 (ObjNum 347 snap 95 seq_num 347) dirty exists 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1048: left oid 25 (ObjNum 347 snap 95 seq_num 347) 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1049: finishing copy_from racing read to smithi02683976-1 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:1049: finishing copy_from to smithi02683976-1 2024-09-13T14:22:28.816 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 264 (ObjNum 296 snap 82 seq_num 296) dirty exists 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: finishing write tid 1 to smithi02683976-24 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: finishing write tid 2 to smithi02683976-24 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: finishing write tid 3 to smithi02683976-24 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: finishing write tid 4 to smithi02683976-24 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 326 (ObjNum 348 snap 95 seq_num 348) dirty exists 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: left oid 24 (ObjNum 348 snap 95 seq_num 348) 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1052: finishing write tid 1 to smithi02683976-5 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1048: done (5 left) 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1049: done (4 left) 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1050: done (3 left) 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1051: done (2 left) 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 95 2024-09-13T14:22:28.817 INFO:tasks.rados.rados.0.smithi026.stdout:1054: seq_num 350 ranges {2220032=671744,2891776=401408} 2024-09-13T14:22:28.832 INFO:tasks.rados.rados.0.smithi026.stdout:1054: writing smithi02683976-35 from 2220032 to 2891776 tid 1 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1054: writing smithi02683976-35 from 2891776 to 3293184 tid 2 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1055: copy_from oid 50 from oid 30 current snap is 95 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1052: finishing write tid 2 to smithi02683976-5 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1053: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1052: finishing write tid 3 to smithi02683976-5 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1053: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1053: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 218 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1052: finishing write tid 4 to smithi02683976-5 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 544 (ObjNum 349 snap 95 seq_num 349) dirty exists 2024-09-13T14:22:28.835 INFO:tasks.rados.rados.0.smithi026.stdout:1052: left oid 5 (ObjNum 349 snap 95 seq_num 349) 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1052: done (3 left) 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1053: done (2 left) 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1056: copy_from oid 41 from oid 3 current snap is 95 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1057: read oid 42 snap -1 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1057: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1058: snap_create 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1054: finishing write tid 1 to smithi02683976-35 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1055: finishing copy_from to smithi02683976-50 2024-09-13T14:22:28.836 INFO:tasks.rados.rados.0.smithi026.stdout:1055: got expected ENOENT (src dne) 2024-09-13T14:22:28.837 INFO:tasks.rados.rados.0.smithi026.stdout:1055: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:22:28.837 INFO:tasks.rados.rados.0.smithi026.stdout:1056: finishing copy_from to smithi02683976-41 2024-09-13T14:22:28.838 INFO:tasks.rados.rados.0.smithi026.stdout:1056: got expected ENOENT (src dne) 2024-09-13T14:22:28.838 INFO:tasks.rados.rados.0.smithi026.stdout:1056: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:22:28.838 INFO:tasks.rados.rados.0.smithi026.stdout:1054: finishing write tid 2 to smithi02683976-35 2024-09-13T14:22:28.838 INFO:tasks.rados.rados.0.smithi026.stdout:1054: finishing write tid 3 to smithi02683976-35 2024-09-13T14:22:28.838 INFO:tasks.rados.rados.0.smithi026.stdout:1057: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:22:28.879 INFO:tasks.rados.rados.0.smithi026.stdout:1054: finishing write tid 4 to smithi02683976-35 2024-09-13T14:22:28.879 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 548 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-13T14:22:28.879 INFO:tasks.rados.rados.0.smithi026.stdout:1054: left oid 35 (ObjNum 350 snap 95 seq_num 350) 2024-09-13T14:22:29.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: pgmap v45: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:29.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[39182]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: pgmap v45: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:29.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:29.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:22:29.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:28 smithi026 ceph-mon[27770]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: pgmap v45: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:22:29.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:29.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:22:29.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:28 smithi145 ceph-mon[31057]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:22:29.836 INFO:tasks.rados.rados.0.smithi026.stdout:1054: done (4 left) 2024-09-13T14:22:29.836 INFO:tasks.rados.rados.0.smithi026.stdout:1055: done (3 left) 2024-09-13T14:22:29.837 INFO:tasks.rados.rados.0.smithi026.stdout:1056: done (2 left) 2024-09-13T14:22:29.837 INFO:tasks.rados.rados.0.smithi026.stdout:1057: done (1 left) 2024-09-13T14:22:29.837 INFO:tasks.rados.rados.0.smithi026.stdout:1058: done (0 left) 2024-09-13T14:22:29.837 INFO:tasks.rados.rados.0.smithi026.stdout:1059: snap_create 2024-09-13T14:22:29.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[27770]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[27770]: Upgrade: Setting container_image for all mgr 2024-09-13T14:22:29.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[27770]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:22:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[39182]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[39182]: Upgrade: Setting container_image for all mgr 2024-09-13T14:22:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:29 smithi026 ceph-mon[39182]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:22:30.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:29 smithi145 ceph-mon[31057]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:30.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:29 smithi145 ceph-mon[31057]: Upgrade: Setting container_image for all mgr 2024-09-13T14:22:30.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:29 smithi145 ceph-mon[31057]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:22:30.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:30.837 INFO:tasks.rados.rados.0.smithi026.stdout:1059: done (0 left) 2024-09-13T14:22:30.837 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 97 2024-09-13T14:22:30.837 INFO:tasks.rados.rados.0.smithi026.stdout:1060: seq_num 351 ranges {1056768=688128,1744896=401408} 2024-09-13T14:22:30.846 INFO:tasks.rados.rados.0.smithi026.stdout:1060: writing smithi02683976-5 from 1056768 to 1744896 tid 1 2024-09-13T14:22:30.849 INFO:tasks.rados.rados.0.smithi026.stdout:1060: writing smithi02683976-5 from 1744896 to 2146304 tid 2 2024-09-13T14:22:30.849 INFO:tasks.rados.rados.0.smithi026.stdout:1061: read oid 22 snap -1 2024-09-13T14:22:30.850 INFO:tasks.rados.rados.0.smithi026.stdout:1061: expect deleted 2024-09-13T14:22:30.850 INFO:tasks.rados.rados.0.smithi026.stdout:1062: rollback oid 47 current snap is 97 2024-09-13T14:22:30.850 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 93 2024-09-13T14:22:30.850 INFO:tasks.rados.rados.0.smithi026.stdout:1063: delete oid 15 current snap is 97 2024-09-13T14:22:30.851 INFO:tasks.rados.rados.0.smithi026.stdout:1060: finishing write tid 1 to smithi02683976-5 2024-09-13T14:22:30.853 INFO:tasks.rados.rados.0.smithi026.stdout:1062: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:1060: finishing write tid 2 to smithi02683976-5 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:1062: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:1060: finishing write tid 3 to smithi02683976-5 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:1062: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 187 (ObjNum 310 snap 84 seq_num 310) dirty exists 2024-09-13T14:22:30.854 INFO:tasks.rados.rados.0.smithi026.stdout:1060: finishing write tid 4 to smithi02683976-5 2024-09-13T14:22:30.855 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 552 (ObjNum 351 snap 97 seq_num 351) dirty exists 2024-09-13T14:22:30.855 INFO:tasks.rados.rados.0.smithi026.stdout:1060: left oid 5 (ObjNum 351 snap 97 seq_num 351) 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:1060: done (3 left) 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:1061: done (2 left) 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:1062: done (1 left) 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:1063: done (0 left) 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:1064: rollback oid 15 current snap is 97 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 96 2024-09-13T14:22:30.857 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 97 2024-09-13T14:22:30.858 INFO:tasks.rados.rados.0.smithi026.stdout:1065: seq_num 353 ranges {0=720896,720896=401408} 2024-09-13T14:22:30.861 INFO:tasks.rados.rados.0.smithi026.stdout:1065: writing smithi02683976-16 from 0 to 720896 tid 1 2024-09-13T14:22:30.863 INFO:tasks.rados.rados.0.smithi026.stdout:1065: writing smithi02683976-16 from 720896 to 1122304 tid 2 2024-09-13T14:22:30.863 INFO:tasks.rados.rados.0.smithi026.stdout:1066: delete oid 19 current snap is 97 2024-09-13T14:22:30.864 INFO:tasks.rados.rados.0.smithi026.stdout:1064: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:22:30.864 INFO:tasks.rados.rados.0.smithi026.stdout:1064: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:22:30.864 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 293 (ObjNum 311 snap 85 seq_num 311) dirty exists 2024-09-13T14:22:30.866 INFO:tasks.rados.rados.0.smithi026.stdout:1065: finishing write tid 1 to smithi02683976-16 2024-09-13T14:22:30.866 INFO:tasks.rados.rados.0.smithi026.stdout:1065: finishing write tid 2 to smithi02683976-16 2024-09-13T14:22:30.867 INFO:tasks.rados.rados.0.smithi026.stdout:1065: finishing write tid 3 to smithi02683976-16 2024-09-13T14:22:30.869 INFO:tasks.rados.rados.0.smithi026.stdout:1065: finishing write tid 4 to smithi02683976-16 2024-09-13T14:22:30.869 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 562 (ObjNum 353 snap 97 seq_num 353) dirty exists 2024-09-13T14:22:30.869 INFO:tasks.rados.rados.0.smithi026.stdout:1065: left oid 16 (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1064: done (2 left) 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1065: done (1 left) 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1066: done (0 left) 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1067: rollback oid 39 current snap is 97 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 93 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1068: read oid 7 snap -1 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1068: expect deleted 2024-09-13T14:22:30.870 INFO:tasks.rados.rados.0.smithi026.stdout:1069: setattr oid 49 current snap is 97 2024-09-13T14:22:30.871 INFO:tasks.rados.rados.0.smithi026.stdout:1070: read oid 15 snap 93 2024-09-13T14:22:30.871 INFO:tasks.rados.rados.0.smithi026.stdout:1070: expect (ObjNum 311 snap 85 seq_num 311) 2024-09-13T14:22:30.871 INFO:tasks.rados.rados.0.smithi026.stdout:1071: snap_create 2024-09-13T14:22:30.873 INFO:tasks.rados.rados.0.smithi026.stdout:1067: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:22:30.874 INFO:tasks.rados.rados.0.smithi026.stdout:1067: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:22:30.874 INFO:tasks.rados.rados.0.smithi026.stdout:1067: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:22:30.874 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 330 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:22:30.875 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 313 (ObjNum 273 snap 70 seq_num 273) dirty exists 2024-09-13T14:22:30.875 INFO:tasks.rados.rados.0.smithi026.stdout:1070: expect (ObjNum 311 snap 85 seq_num 311) 2024-09-13T14:22:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: pgmap v48: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:22:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[39182]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: pgmap v48: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:30 smithi026 ceph-mon[27770]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: pgmap v48: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='mgr.y' 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:31.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:31.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:30 smithi145 ceph-mon[31057]: from='mgr.25022 172.21.15.26:0/1246283297' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:31.639 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:22:31] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1067: done (4 left) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1068: done (3 left) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1069: done (2 left) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1070: done (1 left) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1071: done (0 left) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1072: read oid 50 snap -1 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1072: expect (ObjNum 318 snap 86 seq_num 318) 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 98 2024-09-13T14:22:31.848 INFO:tasks.rados.rados.0.smithi026.stdout:1073: seq_num 355 ranges {5447680=753664,6201344=401408} 2024-09-13T14:22:31.881 INFO:tasks.rados.rados.0.smithi026.stdout:1073: writing smithi02683976-37 from 5447680 to 6201344 tid 1 2024-09-13T14:22:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:1073: writing smithi02683976-37 from 6201344 to 6602752 tid 2 2024-09-13T14:22:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:1074: snap_create 2024-09-13T14:22:31.884 INFO:tasks.rados.rados.0.smithi026.stdout:1072: expect (ObjNum 318 snap 86 seq_num 318) 2024-09-13T14:22:31.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[39182]: Upgrade: Updating mon.a 2024-09-13T14:22:31.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[39182]: Deploying daemon mon.a on smithi026 2024-09-13T14:22:31.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[39182]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:22:31.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[27770]: Upgrade: Updating mon.a 2024-09-13T14:22:31.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[27770]: Deploying daemon mon.a on smithi026 2024-09-13T14:22:31.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:31 smithi026 ceph-mon[27770]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:22:31.934 INFO:tasks.rados.rados.0.smithi026.stdout:1073: finishing write tid 1 to smithi02683976-37 2024-09-13T14:22:31.935 INFO:tasks.rados.rados.0.smithi026.stdout:1073: finishing write tid 2 to smithi02683976-37 2024-09-13T14:22:31.935 INFO:tasks.rados.rados.0.smithi026.stdout:1073: finishing write tid 3 to smithi02683976-37 2024-09-13T14:22:31.935 INFO:tasks.rados.rados.0.smithi026.stdout:1073: finishing write tid 4 to smithi02683976-37 2024-09-13T14:22:31.935 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 293 (ObjNum 355 snap 98 seq_num 355) dirty exists 2024-09-13T14:22:31.935 INFO:tasks.rados.rados.0.smithi026.stdout:1073: left oid 37 (ObjNum 355 snap 98 seq_num 355) 2024-09-13T14:22:32.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:31 smithi026 systemd[1]: Stopping Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:32.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:31 smithi145 ceph-mon[31057]: Upgrade: Updating mon.a 2024-09-13T14:22:32.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:31 smithi145 ceph-mon[31057]: Deploying daemon mon.a on smithi026 2024-09-13T14:22:32.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:31 smithi145 ceph-mon[31057]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:22:32.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[27747]: 2024-09-13T14:22:32.194+0000 7f0da0a5e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:22:32.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:32 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[27747]: 2024-09-13T14:22:32.194+0000 7f0da0a5e640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T14:22:32.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:32 smithi026 podman[96829]: 2024-09-13 14:22:32.439643685 +0000 UTC m=+0.354719070 container died 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T14:22:33.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:32 smithi026 podman[96829]: 2024-09-13 14:22:32.800459999 +0000 UTC m=+0.715535390 container cleanup 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T14:22:33.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:32 smithi026 bash[96829]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:22:33.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 podman[96846]: 2024-09-13 14:22:33.051991356 +0000 UTC m=+0.607627801 container remove 4970e40c75ebe12477cb2ed6d32d64dd3758b652fd719a64b240168515175755 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa) 2024-09-13T14:22:33.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:22:33.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 systemd[1]: Stopped Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:33.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service: Consumed 15.513s CPU time. 2024-09-13T14:22:33.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 systemd[1]: Starting Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 podman[96983]: 2024-09-13 14:22:33.50128818 +0000 UTC m=+0.016152336 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 podman[96983]: 2024-09-13 14:22:33.602848143 +0000 UTC m=+0.117712341 container create fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 podman[96983]: 2024-09-13 14:22:33.822121893 +0000 UTC m=+0.336986050 container init fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 podman[96983]: 2024-09-13 14:22:33.827010026 +0000 UTC m=+0.341874183 container start fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: pidfile_write: ignore empty --pid-file 2024-09-13T14:22:34.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: load: jerasure load: lrc 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Git sha 0 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: DB SUMMARY 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: DB Session ID: 5M8QAQX3R2CRFKKR4FWC 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1173 Bytes 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 5329770 ; 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:22:34.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.env: 0x55db524d9160 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.info_log: 0x55db54375880 2024-09-13T14:22:34.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:22:34.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.statistics: (nil) 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.use_fsync: 0 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:22:34.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.db_log_dir: 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.wal_dir: 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:22:34.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.write_buffer_manager: 0x55db54378960 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:22:34.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.unordered_write: 0 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:22:34.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.row_cache: None 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.wal_filter: None 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.wal_compression: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:22:34.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:22:34.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:22:34.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_open_files: -1 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Compression algorithms supported: 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kZSTD supported: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:22:34.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.merge_operator: 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_filter: None 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:22:34.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55db543754e0) 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 4 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x55db54368850 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:22:34.179 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-09-13T14:22:34.180 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: bloomfilter 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 5 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_auto_readahead_size: 262144 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: prepopulate_block_cache: 0 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:22:34.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression: NoCompression 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.num_levels: 7 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:34.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:34.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:22:34.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:22:34.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:22:34.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:22:34.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:22:34.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:22:34.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:22:34.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:22:34.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:22:34.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.ttl: 2592000 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:22:34.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12859, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ebdadac2-46c1-40d4-a3dd-15131196942b 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237353854351, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-13T14:22:34.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237353869730, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 4999581, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12860, "largest_seqno": 15821, "table_properties": {"data_size": 4985066, "index_size": 9884, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3589, "raw_key_size": 32556, "raw_average_key_size": 22, "raw_value_size": 4955207, "raw_average_value_size": 3496, "num_data_blocks": 430, "num_entries": 1417, "num_filter_entries": 1417, "num_deletions": 310, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237353, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ebdadac2-46c1-40d4-a3dd-15131196942b", "db_session_id": "5M8QAQX3R2CRFKKR4FWC", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237353869863, "job": 1, "event": "recovery_finished"} 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55db54398e00 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: DB pointer 0x55db54498000 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:34.191 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 1/0 4.77 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 655.7 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: L6 1/0 7.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 12.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 655.7 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 655.7 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:22:34.192 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 655.7 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:34.193 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 263.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 263.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache BinnedLRUCache@0x55db54368850#7 capacity: 512.00 MB usage: 14.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,10.08 KB,0.00192225%) Misc(1,0.00 KB,0%) 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???) e3 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).mds e8 new map 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).mds e8 print_map 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: e8 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:22:34.194 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: 1 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: Filesystem 'cephfs' (1) 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: fs_name cephfs 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: epoch 8 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: created 2024-09-13T14:15:28.665431+0000 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: modified 2024-09-13T14:15:41.581255+0000 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: tableserver 0 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: root 0 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_timeout 60 2024-09-13T14:22:34.195 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_autoclose 300 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_file_size 1099511627776 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_xattr_size 65536 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: required_client_features {} 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure 0 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure_osd_epoch 0 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_mds 1 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: in 0 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: up {0=24331} 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: failed 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: damaged 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: stopped 2024-09-13T14:22:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_pools [3] 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_pool 2 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: inline_data disabled 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: balancer 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: bal_rank_mask -1 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: standby_count_wanted 0 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.b{0:24331} state up:active seq 6 addr [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Standby daemons: 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.a{-1:14421} state up:standby seq 1 addr [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).osd e265 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-13T14:22:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:34.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:34.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:34.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 ceph-mon[97022]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-13T14:22:34.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 bash[96983]: fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af 2024-09-13T14:22:34.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:33 smithi026 systemd[1]: Started Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:34.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:35.358 INFO:tasks.rados.rados.0.smithi026.stdout:1072: done (2 left) 2024-09-13T14:22:35.358 INFO:tasks.rados.rados.0.smithi026.stdout:1073: done (1 left) 2024-09-13T14:22:35.358 INFO:tasks.rados.rados.0.smithi026.stdout:1074: done (0 left) 2024-09-13T14:22:35.358 INFO:tasks.rados.rados.0.smithi026.stdout:1075: copy_from oid 12 from oid 23 current snap is 99 2024-09-13T14:22:35.359 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 99 2024-09-13T14:22:35.359 INFO:tasks.rados.rados.0.smithi026.stdout:1076: seq_num 356 ranges {0=770048,770048=401408} 2024-09-13T14:22:35.362 INFO:tasks.rados.rados.0.smithi026.stdout:1076: writing smithi02683976-19 from 0 to 770048 tid 1 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1076: writing smithi02683976-19 from 770048 to 1171456 tid 2 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1077: read oid 2 snap -1 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1077: expect (ObjNum 328 snap 89 seq_num 328) 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1078: read oid 14 snap -1 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1078: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1079: copy_from oid 31 from oid 17 current snap is 99 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1080: read oid 5 snap -1 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1080: expect (ObjNum 351 snap 97 seq_num 351) 2024-09-13T14:22:35.366 INFO:tasks.rados.rados.0.smithi026.stdout:1081: snap_remove snap 93 2024-09-13T14:22:35.632 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: ignoring --setuser ceph since I am not root 2024-09-13T14:22:35.632 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: ignoring --setgroup ceph since I am not root 2024-09-13T14:22:35.632 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mgr[80590]: -- 172.21.15.145:0/3172708148 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ec57c330e0 con 0x55ec57c11400 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: pgmap v53: 121 pgs: 121 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: mon.a calling monitor election 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: monmap epoch 3 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: min_mon_release 17 (quincy) 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: election_strategy: 1 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:35.633 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: overall HEALTH_OK 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: from='mgr.25022 ' entity='' 2024-09-13T14:22:35.634 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-mon[31057]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-13T14:22:35.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ignoring --setuser ceph since I am not root 2024-09-13T14:22:35.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ignoring --setgroup ceph since I am not root 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: pgmap v53: 121 pgs: 121 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: mon.a calling monitor election 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: monmap epoch 3 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: min_mon_release 17 (quincy) 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: election_strategy: 1 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: overall HEALTH_OK 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: from='mgr.25022 ' entity='' 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[97022]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: pgmap v53: 121 pgs: 121 active+clean; 141 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: mon.a calling monitor election 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: monmap epoch 3 2024-09-13T14:22:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: min_mon_release 17 (quincy) 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: election_strategy: 1 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: overall HEALTH_OK 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: from='mgr.25022 ' entity='' 2024-09-13T14:22:35.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:35 smithi026 ceph-mon[39182]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-13T14:22:35.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:35.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:36.362 INFO:tasks.rados.rados.0.smithi026.stdout:1081: done (6 left) 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1082: snap_create 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: finishing write tid 1 to smithi02683976-19 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: finishing write tid 2 to smithi02683976-19 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: finishing write tid 3 to smithi02683976-19 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: finishing write tid 4 to smithi02683976-19 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 567 (ObjNum 356 snap 99 seq_num 356) dirty exists 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: left oid 19 (ObjNum 356 snap 99 seq_num 356) 2024-09-13T14:22:36.363 INFO:tasks.rados.rados.0.smithi026.stdout:1077: expect (ObjNum 328 snap 89 seq_num 328) 2024-09-13T14:22:36.386 INFO:tasks.rados.rados.0.smithi026.stdout:1079: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:22:36.386 INFO:tasks.rados.rados.0.smithi026.stdout:1079: finishing copy_from to smithi02683976-31 2024-09-13T14:22:36.386 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 387 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:22:36.386 INFO:tasks.rados.rados.0.smithi026.stdout:1080: expect (ObjNum 351 snap 97 seq_num 351) 2024-09-13T14:22:36.460 INFO:tasks.rados.rados.0.smithi026.stdout:1075: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:22:36.460 INFO:tasks.rados.rados.0.smithi026.stdout:1078: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:22:36.631 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:36 smithi145 ceph-mon[31057]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:22:36.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:36 smithi026 ceph-mon[39182]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:22:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:36 smithi026 ceph-mon[97022]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:22:36.795 INFO:tasks.rados.rados.0.smithi026.stdout:1075: finishing copy_from to smithi02683976-12 2024-09-13T14:22:36.796 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 290 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:22:37.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:36 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1075: done (6 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1076: done (5 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1077: done (4 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1078: done (3 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1079: done (2 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1080: done (1 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1082: done (0 left) 2024-09-13T14:22:37.363 INFO:tasks.rados.rados.0.smithi026.stdout:1083: read oid 29 snap -1 2024-09-13T14:22:37.364 INFO:tasks.rados.rados.0.smithi026.stdout:1083: expect deleted 2024-09-13T14:22:37.364 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 100 2024-09-13T14:22:37.364 INFO:tasks.rados.rados.0.smithi026.stdout:1084: seq_num 357 ranges {8413184=786432,9199616=401408} 2024-09-13T14:22:37.413 INFO:tasks.rados.rados.0.smithi026.stdout:1084: writing smithi02683976-6 from 8413184 to 9199616 tid 1 2024-09-13T14:22:37.415 INFO:tasks.rados.rados.0.smithi026.stdout:1084: writing smithi02683976-6 from 9199616 to 9601024 tid 2 2024-09-13T14:22:37.415 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 100 2024-09-13T14:22:37.415 INFO:tasks.rados.rados.0.smithi026.stdout:1085: seq_num 358 ranges {4603904=802816,5406720=401408} 2024-09-13T14:22:37.444 INFO:tasks.rados.rados.0.smithi026.stdout:1085: writing smithi02683976-24 from 4603904 to 5406720 tid 1 2024-09-13T14:22:37.447 INFO:tasks.rados.rados.0.smithi026.stdout:1085: writing smithi02683976-24 from 5406720 to 5808128 tid 2 2024-09-13T14:22:37.447 INFO:tasks.rados.rados.0.smithi026.stdout:1086: delete oid 33 current snap is 100 2024-09-13T14:22:37.447 INFO:tasks.rados.rados.0.smithi026.stdout:1084: finishing write tid 1 to smithi02683976-6 2024-09-13T14:22:37.448 INFO:tasks.rados.rados.0.smithi026.stdout:1084: finishing write tid 2 to smithi02683976-6 2024-09-13T14:22:37.448 INFO:tasks.rados.rados.0.smithi026.stdout:1084: finishing write tid 3 to smithi02683976-6 2024-09-13T14:22:37.448 INFO:tasks.rados.rados.0.smithi026.stdout:1084: finishing write tid 4 to smithi02683976-6 2024-09-13T14:22:37.448 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 623 (ObjNum 357 snap 100 seq_num 357) dirty exists 2024-09-13T14:22:37.448 INFO:tasks.rados.rados.0.smithi026.stdout:1084: left oid 6 (ObjNum 357 snap 100 seq_num 357) 2024-09-13T14:22:37.450 INFO:tasks.rados.rados.0.smithi026.stdout:1083: done (3 left) 2024-09-13T14:22:37.450 INFO:tasks.rados.rados.0.smithi026.stdout:1084: done (2 left) 2024-09-13T14:22:37.450 INFO:tasks.rados.rados.0.smithi026.stdout:1086: done (1 left) 2024-09-13T14:22:37.450 INFO:tasks.rados.rados.0.smithi026.stdout:1087: rollback oid 38 current snap is 100 2024-09-13T14:22:37.451 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 98 2024-09-13T14:22:37.451 INFO:tasks.rados.rados.0.smithi026.stdout:1088: delete oid 3 current snap is 100 2024-09-13T14:22:37.451 INFO:tasks.rados.rados.0.smithi026.stdout:1085: finishing write tid 1 to smithi02683976-24 2024-09-13T14:22:37.451 INFO:tasks.rados.rados.0.smithi026.stdout:1085: finishing write tid 2 to smithi02683976-24 2024-09-13T14:22:37.452 INFO:tasks.rados.rados.0.smithi026.stdout:1085: finishing write tid 3 to smithi02683976-24 2024-09-13T14:22:37.452 INFO:tasks.rados.rados.0.smithi026.stdout:1088: done (2 left) 2024-09-13T14:22:37.452 INFO:tasks.rados.rados.0.smithi026.stdout:1089: snap_create 2024-09-13T14:22:37.453 INFO:tasks.rados.rados.0.smithi026.stdout:1085: finishing write tid 4 to smithi02683976-24 2024-09-13T14:22:37.453 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 340 (ObjNum 358 snap 100 seq_num 358) dirty exists 2024-09-13T14:22:37.453 INFO:tasks.rados.rados.0.smithi026.stdout:1085: left oid 24 (ObjNum 358 snap 100 seq_num 358) 2024-09-13T14:22:37.454 INFO:tasks.rados.rados.0.smithi026.stdout:1087: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:22:37.454 INFO:tasks.rados.rados.0.smithi026.stdout:1087: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:22:37.454 INFO:tasks.rados.rados.0.smithi026.stdout:1087: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:22:37.454 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 483 (ObjNum 333 snap 91 seq_num 333) dirty exists 2024-09-13T14:22:37.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:37 smithi026 ceph-mon[97022]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:22:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:37 smithi026 ceph-mon[39182]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:22:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:37 smithi145 ceph-mon[31057]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:22:38.370 INFO:tasks.rados.rados.0.smithi026.stdout:1085: done (2 left) 2024-09-13T14:22:38.370 INFO:tasks.rados.rados.0.smithi026.stdout:1087: done (1 left) 2024-09-13T14:22:38.370 INFO:tasks.rados.rados.0.smithi026.stdout:1089: done (0 left) 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:1090: read oid 12 snap -1 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:1090: expect (ObjNum 336 snap 91 seq_num 336) 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:1091: rollback oid 47 current snap is 101 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 95 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:1092: copy_from oid 22 from oid 35 current snap is 101 2024-09-13T14:22:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:1093: delete oid 3 current snap is 101 2024-09-13T14:22:38.373 INFO:tasks.rados.rados.0.smithi026.stdout:1093: done (3 left) 2024-09-13T14:22:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:1094: rmattr oid 24 current snap is 101 2024-09-13T14:22:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:1094: done (3 left) 2024-09-13T14:22:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 101 2024-09-13T14:22:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:1095: seq_num 362 ranges {1425408=466944,1892352=466944,2359296=335872} 2024-09-13T14:22:38.385 INFO:tasks.rados.rados.0.smithi026.stdout:1095: writing smithi02683976-42 from 1425408 to 1892352 tid 1 2024-09-13T14:22:38.388 INFO:tasks.rados.rados.0.smithi026.stdout:1095: writing smithi02683976-42 from 1892352 to 2359296 tid 2 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:1095: writing smithi02683976-42 from 2359296 to 2695168 tid 3 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:1096: snap_create 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:1091: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:1091: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:1091: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:22:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 193 (ObjNum 310 snap 84 seq_num 310) dirty exists 2024-09-13T14:22:38.393 INFO:tasks.rados.rados.0.smithi026.stdout:1095: finishing write tid 1 to smithi02683976-42 2024-09-13T14:22:38.393 INFO:tasks.rados.rados.0.smithi026.stdout:1090: expect (ObjNum 336 snap 91 seq_num 336) 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1095: finishing write tid 2 to smithi02683976-42 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1092: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1095: finishing write tid 3 to smithi02683976-42 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1095: finishing write tid 4 to smithi02683976-42 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1092: finishing copy_from to smithi02683976-22 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 548 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-13T14:22:38.580 INFO:tasks.rados.rados.0.smithi026.stdout:1095: finishing write tid 5 to smithi02683976-42 2024-09-13T14:22:38.581 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 297 (ObjNum 362 snap 101 seq_num 362) dirty exists 2024-09-13T14:22:38.581 INFO:tasks.rados.rados.0.smithi026.stdout:1095: left oid 42 (ObjNum 362 snap 101 seq_num 362) 2024-09-13T14:22:38.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:38 smithi026 ceph-mon[39182]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:22:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:38 smithi026 ceph-mon[97022]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:22:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:38 smithi145 ceph-mon[31057]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1090: done (4 left) 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1091: done (3 left) 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1092: done (2 left) 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1095: done (1 left) 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1096: done (0 left) 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 102 2024-09-13T14:22:38.882 INFO:tasks.rados.rados.0.smithi026.stdout:1097: seq_num 363 ranges {1310720=483328,1794048=753664,2547712=49152} 2024-09-13T14:22:38.894 INFO:tasks.rados.rados.0.smithi026.stdout:1097: writing smithi02683976-39 from 1310720 to 1794048 tid 1 2024-09-13T14:22:38.898 INFO:tasks.rados.rados.0.smithi026.stdout:1097: writing smithi02683976-39 from 1794048 to 2547712 tid 2 2024-09-13T14:22:38.898 INFO:tasks.rados.rados.0.smithi026.stdout:1097: writing smithi02683976-39 from 2547712 to 2596864 tid 3 2024-09-13T14:22:38.898 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 102 2024-09-13T14:22:38.899 INFO:tasks.rados.rados.0.smithi026.stdout:1097: finishing write tid 1 to smithi02683976-39 2024-09-13T14:22:38.899 INFO:tasks.rados.rados.0.smithi026.stdout:1098: seq_num 364 ranges {1867776=499712,2367488=638976,3006464=163840} 2024-09-13T14:22:38.912 INFO:tasks.rados.rados.0.smithi026.stdout:1098: writing smithi02683976-43 from 1867776 to 2367488 tid 1 2024-09-13T14:22:38.917 INFO:tasks.rados.rados.0.smithi026.stdout:1098: writing smithi02683976-43 from 2367488 to 3006464 tid 2 2024-09-13T14:22:38.918 INFO:tasks.rados.rados.0.smithi026.stdout:1098: writing smithi02683976-43 from 3006464 to 3170304 tid 3 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1099: read oid 2 snap -1 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1099: expect (ObjNum 328 snap 89 seq_num 328) 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 102 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1097: finishing write tid 2 to smithi02683976-39 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1097: finishing write tid 3 to smithi02683976-39 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1097: finishing write tid 4 to smithi02683976-39 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1097: finishing write tid 5 to smithi02683976-39 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 345 (ObjNum 363 snap 102 seq_num 363) dirty exists 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1097: left oid 39 (ObjNum 363 snap 102 seq_num 363) 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1098: finishing write tid 1 to smithi02683976-43 2024-09-13T14:22:38.919 INFO:tasks.rados.rados.0.smithi026.stdout:1100: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2024-09-13T14:22:38.922 INFO:tasks.rados.rados.0.smithi026.stdout:1100: writing smithi02683976-11 from 0 to 516096 tid 1 2024-09-13T14:22:38.926 INFO:tasks.rados.rados.0.smithi026.stdout:1100: writing smithi02683976-11 from 516096 to 1089536 tid 2 2024-09-13T14:22:38.928 INFO:tasks.rados.rados.0.smithi026.stdout:1100: writing smithi02683976-11 from 1089536 to 1318912 tid 3 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1097: done (3 left) 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1101: rmattr oid 48 current snap is 102 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1098: finishing write tid 2 to smithi02683976-43 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1098: finishing write tid 3 to smithi02683976-43 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1098: finishing write tid 4 to smithi02683976-43 2024-09-13T14:22:38.929 INFO:tasks.rados.rados.0.smithi026.stdout:1099: expect (ObjNum 328 snap 89 seq_num 328) 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1098: finishing write tid 5 to smithi02683976-43 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 574 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1098: left oid 43 (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: finishing write tid 1 to smithi02683976-11 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: finishing write tid 2 to smithi02683976-11 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: finishing write tid 3 to smithi02683976-11 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: finishing write tid 4 to smithi02683976-11 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: finishing write tid 5 to smithi02683976-11 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 627 (ObjNum 365 snap 102 seq_num 365) dirty exists 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1100: left oid 11 (ObjNum 365 snap 102 seq_num 365) 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 195 (ObjNum 280 snap 71 seq_num 280) dirty exists 2024-09-13T14:22:38.949 INFO:tasks.rados.rados.0.smithi026.stdout:1098: done (3 left) 2024-09-13T14:22:38.950 INFO:tasks.rados.rados.0.smithi026.stdout:1099: done (2 left) 2024-09-13T14:22:38.950 INFO:tasks.rados.rados.0.smithi026.stdout:1100: done (1 left) 2024-09-13T14:22:38.950 INFO:tasks.rados.rados.0.smithi026.stdout:1101: done (0 left) 2024-09-13T14:22:38.950 INFO:tasks.rados.rados.0.smithi026.stdout:1102: rmattr oid 22 current snap is 102 2024-09-13T14:22:38.950 INFO:tasks.rados.rados.0.smithi026.stdout:1103: snap_remove snap 96 2024-09-13T14:22:39.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:39 smithi026 ceph-mon[39182]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:22:39.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:39 smithi026 ceph-mon[39182]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:22:39.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:39 smithi026 ceph-mon[97022]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:22:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:39 smithi026 ceph-mon[97022]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:22:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:39 smithi145 ceph-mon[31057]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:22:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:39 smithi145 ceph-mon[31057]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:22:39.883 INFO:tasks.rados.rados.0.smithi026.stdout:1103: done (1 left) 2024-09-13T14:22:39.883 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 102 2024-09-13T14:22:39.884 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 0 (ObjNum 350 snap 95 seq_num 350) dirty exists 2024-09-13T14:22:39.884 INFO:tasks.rados.rados.0.smithi026.stdout:1104: seq_num 366 ranges {0=532480,532480=458752,991232=344064} 2024-09-13T14:22:39.887 INFO:tasks.rados.rados.0.smithi026.stdout:1104: writing smithi02683976-30 from 0 to 532480 tid 1 2024-09-13T14:22:39.890 INFO:tasks.rados.rados.0.smithi026.stdout:1104: writing smithi02683976-30 from 532480 to 991232 tid 2 2024-09-13T14:22:39.893 INFO:tasks.rados.rados.0.smithi026.stdout:1104: writing smithi02683976-30 from 991232 to 1335296 tid 3 2024-09-13T14:22:39.893 INFO:tasks.rados.rados.0.smithi026.stdout:1102: done (1 left) 2024-09-13T14:22:39.893 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 102 2024-09-13T14:22:39.893 INFO:tasks.rados.rados.0.smithi026.stdout:1105: seq_num 367 ranges {3293184=548864,3842048=737280,4579328=65536} 2024-09-13T14:22:39.916 INFO:tasks.rados.rados.0.smithi026.stdout:1105: writing smithi02683976-22 from 3293184 to 3842048 tid 1 2024-09-13T14:22:39.921 INFO:tasks.rados.rados.0.smithi026.stdout:1105: writing smithi02683976-22 from 3842048 to 4579328 tid 2 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1105: writing smithi02683976-22 from 4579328 to 4644864 tid 3 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1106: snap_create 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: finishing write tid 1 to smithi02683976-30 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: finishing write tid 2 to smithi02683976-30 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: finishing write tid 3 to smithi02683976-30 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: finishing write tid 4 to smithi02683976-30 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: finishing write tid 5 to smithi02683976-30 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 297 (ObjNum 366 snap 102 seq_num 366) dirty exists 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1104: left oid 30 (ObjNum 366 snap 102 seq_num 366) 2024-09-13T14:22:39.922 INFO:tasks.rados.rados.0.smithi026.stdout:1105: finishing write tid 1 to smithi02683976-22 2024-09-13T14:22:39.925 INFO:tasks.rados.rados.0.smithi026.stdout:1105: finishing write tid 2 to smithi02683976-22 2024-09-13T14:22:39.925 INFO:tasks.rados.rados.0.smithi026.stdout:1105: finishing write tid 3 to smithi02683976-22 2024-09-13T14:22:39.925 INFO:tasks.rados.rados.0.smithi026.stdout:1105: finishing write tid 4 to smithi02683976-22 2024-09-13T14:22:39.928 INFO:tasks.rados.rados.0.smithi026.stdout:1105: finishing write tid 5 to smithi02683976-22 2024-09-13T14:22:39.928 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 552 (ObjNum 367 snap 102 seq_num 367) dirty exists 2024-09-13T14:22:39.929 INFO:tasks.rados.rados.0.smithi026.stdout:1105: left oid 22 (ObjNum 367 snap 102 seq_num 367) 2024-09-13T14:22:40.489 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:40.489 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:40] ENGINE Bus STARTING 2024-09-13T14:22:40.490 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: CherryPy Checker: 2024-09-13T14:22:40.490 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: The Application mounted at '' has an empty config. 2024-09-13T14:22:40.490 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: 2024-09-13T14:22:40.688 INFO:tasks.rados.rados.0.smithi026.stdout:1104: done (2 left) 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1105: done (1 left) 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1106: done (0 left) 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1107: read oid 5 snap 91 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1107: expect deleted 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1108: read oid 47 snap 98 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1108: expect (ObjNum 310 snap 84 seq_num 310) 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1109: copy_from oid 23 from oid 15 current snap is 103 2024-09-13T14:22:40.689 INFO:tasks.rados.rados.0.smithi026.stdout:1110: delete oid 9 current snap is 103 2024-09-13T14:22:40.691 INFO:tasks.rados.rados.0.smithi026.stdout:1108: expect (ObjNum 310 snap 84 seq_num 310) 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1109: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1109: finishing copy_from to smithi02683976-23 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 293 (ObjNum 311 snap 85 seq_num 311) dirty exists 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1107: done (3 left) 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1108: done (2 left) 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1109: done (1 left) 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1110: done (0 left) 2024-09-13T14:22:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:1111: setattr oid 40 current snap is 103 2024-09-13T14:22:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:1112: rollback oid 26 current snap is 103 2024-09-13T14:22:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 95 2024-09-13T14:22:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:1113: delete oid 42 current snap is 103 2024-09-13T14:22:40.712 INFO:tasks.rados.rados.0.smithi026.stdout:1113: done (2 left) 2024-09-13T14:22:40.712 INFO:tasks.rados.rados.0.smithi026.stdout:1114: copy_from oid 49 from oid 42 current snap is 103 2024-09-13T14:22:40.712 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 103 2024-09-13T14:22:40.712 INFO:tasks.rados.rados.0.smithi026.stdout:1115: seq_num 370 ranges {2146304=606208,2752512=794624,3547136=8192} 2024-09-13T14:22:40.727 INFO:tasks.rados.rados.0.smithi026.stdout:1115: writing smithi02683976-5 from 2146304 to 2752512 tid 1 2024-09-13T14:22:40.732 INFO:tasks.rados.rados.0.smithi026.stdout:1115: writing smithi02683976-5 from 2752512 to 3547136 tid 2 2024-09-13T14:22:40.732 INFO:tasks.rados.rados.0.smithi026.stdout:1115: writing smithi02683976-5 from 3547136 to 3555328 tid 3 2024-09-13T14:22:40.732 INFO:tasks.rados.rados.0.smithi026.stdout:1116: delete oid 48 current snap is 103 2024-09-13T14:22:40.732 INFO:tasks.rados.rados.0.smithi026.stdout:1112: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:22:40.732 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 319 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1112: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1112: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 555 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1114: finishing copy_from to smithi02683976-49 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1114: got expected ENOENT (src dne) 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1114: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:22:40.733 INFO:tasks.rados.rados.0.smithi026.stdout:1115: finishing write tid 1 to smithi02683976-5 2024-09-13T14:22:40.735 INFO:tasks.rados.rados.0.smithi026.stdout:1111: done (4 left) 2024-09-13T14:22:40.735 INFO:tasks.rados.rados.0.smithi026.stdout:1112: done (3 left) 2024-09-13T14:22:40.735 INFO:tasks.rados.rados.0.smithi026.stdout:1114: done (2 left) 2024-09-13T14:22:40.735 INFO:tasks.rados.rados.0.smithi026.stdout:1116: done (1 left) 2024-09-13T14:22:40.735 INFO:tasks.rados.rados.0.smithi026.stdout:1117: snap_create 2024-09-13T14:22:40.736 INFO:tasks.rados.rados.0.smithi026.stdout:1115: finishing write tid 2 to smithi02683976-5 2024-09-13T14:22:40.736 INFO:tasks.rados.rados.0.smithi026.stdout:1115: finishing write tid 3 to smithi02683976-5 2024-09-13T14:22:40.736 INFO:tasks.rados.rados.0.smithi026.stdout:1115: finishing write tid 4 to smithi02683976-5 2024-09-13T14:22:40.736 INFO:tasks.rados.rados.0.smithi026.stdout:1115: finishing write tid 5 to smithi02683976-5 2024-09-13T14:22:40.736 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 567 (ObjNum 370 snap 103 seq_num 370) dirty exists 2024-09-13T14:22:40.737 INFO:tasks.rados.rados.0.smithi026.stdout:1115: left oid 5 (ObjNum 370 snap 103 seq_num 370) 2024-09-13T14:22:40.881 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:40] ENGINE Serving on http://:::9283 2024-09-13T14:22:40.881 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x[80567]: [13/Sep/2024:14:22:40] ENGINE Bus STARTED 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: Standby manager daemon x restarted 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: Standby manager daemon x started 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:41.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: Active manager daemon y restarted 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: Activating manager daemon y 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: mgrmap e38: y(active, starting, since 0.00681765s), standbys: x 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[97022]: Manager daemon y is now available 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:22:41.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: Standby manager daemon x restarted 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: Standby manager daemon x started 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: Active manager daemon y restarted 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: Activating manager daemon y 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: mgrmap e38: y(active, starting, since 0.00681765s), standbys: x 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:41.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:41.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:41.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:40 smithi026 ceph-mon[39182]: Manager daemon y is now available 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: Standby manager daemon x restarted 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: Standby manager daemon x started 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.? 172.21.15.145:0/2337941013' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: Active manager daemon y restarted 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: Activating manager daemon y 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:22:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: mgrmap e38: y(active, starting, since 0.00681765s), standbys: x 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:22:41.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:22:41.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:22:41.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:22:41.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:22:41.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:22:41.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:40 smithi145 ceph-mon[31057]: Manager daemon y is now available 2024-09-13T14:22:41.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.221+0000 7f82ae6e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.221+0000 7f82ae6e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.221+0000 7f82ae6e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.221+0000 7f82ae6e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.476 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.221+0000 7f82ae6e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.222+0000 7f82abee1640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.222+0000 7f82abee1640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.222+0000 7f82abee1640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.222+0000 7f82abee1640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.477 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.222+0000 7f82abee1640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:22:41.702 INFO:tasks.rados.rados.0.smithi026.stdout:1115: done (1 left) 2024-09-13T14:22:41.702 INFO:tasks.rados.rados.0.smithi026.stdout:1117: done (0 left) 2024-09-13T14:22:41.702 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 104 2024-09-13T14:22:41.702 INFO:tasks.rados.rados.0.smithi026.stdout:1118: seq_num 372 ranges {401408=638976,1040384=565248,1605632=237568} 2024-09-13T14:22:41.707 INFO:tasks.rados.rados.0.smithi026.stdout:1118: writing smithi02683976-47 from 401408 to 1040384 tid 1 2024-09-13T14:22:41.711 INFO:tasks.rados.rados.0.smithi026.stdout:1118: writing smithi02683976-47 from 1040384 to 1605632 tid 2 2024-09-13T14:22:41.713 INFO:tasks.rados.rados.0.smithi026.stdout:1118: writing smithi02683976-47 from 1605632 to 1843200 tid 3 2024-09-13T14:22:41.713 INFO:tasks.rados.rados.0.smithi026.stdout:1119: snap_remove snap 103 2024-09-13T14:22:41.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:41.693+0000 7f82de253640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T14:22:41.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:41] ENGINE Bus STARTING 2024-09-13T14:22:41.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: CherryPy Checker: 2024-09-13T14:22:41.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: The Application mounted at '' has an empty config. 2024-09-13T14:22:41.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:42.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: mgrmap e39: y(active, since 1.01175s), standbys: x 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[39182]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: mgrmap e39: y(active, since 1.01175s), standbys: x 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-mon[97022]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:41] ENGINE Serving on http://:::9283 2024-09-13T14:22:42.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:41] ENGINE Bus STARTED 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: mgrmap e39: y(active, since 1.01175s), standbys: x 2024-09-13T14:22:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:41 smithi145 ceph-mon[31057]: osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:22:42.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:42.361+0000 7f828c528640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:22:42] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:42.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: Traceback (most recent call last): 2024-09-13T14:22:42.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:42.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._connections.run(self.expiration_interval) 2024-09-13T14:22:42.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._run(expiration_interval) 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: s = self.context.wrap_socket( 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: return self.sslsocket_class._create( 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self.do_handshake() 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:42.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: self._sslobj.do_handshake() 2024-09-13T14:22:42.668 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:42.668 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1119: done (1 left) 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1120: delete oid 9 current snap is 104 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: finishing write tid 1 to smithi02683976-47 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: finishing write tid 2 to smithi02683976-47 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: finishing write tid 3 to smithi02683976-47 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: finishing write tid 4 to smithi02683976-47 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: finishing write tid 5 to smithi02683976-47 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 206 (ObjNum 372 snap 104 seq_num 372) dirty exists 2024-09-13T14:22:42.705 INFO:tasks.rados.rados.0.smithi026.stdout:1118: left oid 47 (ObjNum 372 snap 104 seq_num 372) 2024-09-13T14:22:42.709 INFO:tasks.rados.rados.0.smithi026.stdout:1118: done (1 left) 2024-09-13T14:22:42.709 INFO:tasks.rados.rados.0.smithi026.stdout:1120: done (0 left) 2024-09-13T14:22:42.709 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 104 2024-09-13T14:22:42.709 INFO:tasks.rados.rados.0.smithi026.stdout:1121: seq_num 374 ranges {0=671744,671744=778240,1449984=24576} 2024-09-13T14:22:42.713 INFO:tasks.rados.rados.0.smithi026.stdout:1121: writing smithi02683976-46 from 0 to 671744 tid 1 2024-09-13T14:22:42.718 INFO:tasks.rados.rados.0.smithi026.stdout:1121: writing smithi02683976-46 from 671744 to 1449984 tid 2 2024-09-13T14:22:42.719 INFO:tasks.rados.rados.0.smithi026.stdout:1121: writing smithi02683976-46 from 1449984 to 1474560 tid 3 2024-09-13T14:22:42.719 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 104 2024-09-13T14:22:42.719 INFO:tasks.rados.rados.0.smithi026.stdout:1121: finishing write tid 1 to smithi02683976-46 2024-09-13T14:22:42.719 INFO:tasks.rados.rados.0.smithi026.stdout:1122: seq_num 375 ranges {417792=688128,1105920=663552,1769472=139264} 2024-09-13T14:22:42.726 INFO:tasks.rados.rados.0.smithi026.stdout:1122: writing smithi02683976-15 from 417792 to 1105920 tid 1 2024-09-13T14:22:42.729 INFO:tasks.rados.rados.0.smithi026.stdout:1122: writing smithi02683976-15 from 1105920 to 1769472 tid 2 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1122: writing smithi02683976-15 from 1769472 to 1908736 tid 3 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1123: read oid 31 snap -1 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1123: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1124: copy_from oid 33 from oid 44 current snap is 104 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: finishing write tid 2 to smithi02683976-46 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: finishing write tid 3 to smithi02683976-46 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: finishing write tid 4 to smithi02683976-46 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: finishing write tid 5 to smithi02683976-46 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 427 (ObjNum 374 snap 104 seq_num 374) dirty exists 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: left oid 46 (ObjNum 374 snap 104 seq_num 374) 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1122: finishing write tid 1 to smithi02683976-15 2024-09-13T14:22:42.731 INFO:tasks.rados.rados.0.smithi026.stdout:1121: done (3 left) 2024-09-13T14:22:42.732 INFO:tasks.rados.rados.0.smithi026.stdout:1125: snap_create 2024-09-13T14:22:42.733 INFO:tasks.rados.rados.0.smithi026.stdout:1124: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:22:42.734 INFO:tasks.rados.rados.0.smithi026.stdout:1122: finishing write tid 2 to smithi02683976-15 2024-09-13T14:22:42.734 INFO:tasks.rados.rados.0.smithi026.stdout:1124: finishing copy_from to smithi02683976-33 2024-09-13T14:22:42.734 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 421 (ObjNum 33 snap 0 seq_num 7) dirty exists 2024-09-13T14:22:42.734 INFO:tasks.rados.rados.0.smithi026.stdout:1122: finishing write tid 3 to smithi02683976-15 2024-09-13T14:22:42.734 INFO:tasks.rados.rados.0.smithi026.stdout:1122: finishing write tid 4 to smithi02683976-15 2024-09-13T14:22:42.735 INFO:tasks.rados.rados.0.smithi026.stdout:1122: finishing write tid 5 to smithi02683976-15 2024-09-13T14:22:42.735 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 301 (ObjNum 375 snap 104 seq_num 375) dirty exists 2024-09-13T14:22:42.735 INFO:tasks.rados.rados.0.smithi026.stdout:1122: left oid 15 (ObjNum 375 snap 104 seq_num 375) 2024-09-13T14:22:42.738 INFO:tasks.rados.rados.0.smithi026.stdout:1123: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:22:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: [13/Sep/2024:14:22:42] ENGINE Bus STARTING 2024-09-13T14:22:43.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: [13/Sep/2024:14:22:42] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: [13/Sep/2024:14:22:42] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: [13/Sep/2024:14:22:42] ENGINE Bus STARTED 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: [13/Sep/2024:14:22:42] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:43.166 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.do_handshake() 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[97022]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:42] ENGINE Bus STARTING 2024-09-13T14:22:43.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:42] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:42] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:42] ENGINE Bus STARTED 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: [13/Sep/2024:14:22:42] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._run(expiration_interval) 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.do_handshake() 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:43.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:42 smithi026 ceph-mon[39182]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:22:43.197 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:42] ENGINE Bus STARTING 2024-09-13T14:22:43.197 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:42] ENGINE Serving on http://172.21.15.26:8765 2024-09-13T14:22:43.197 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:42] ENGINE Serving on https://172.21.15.26:7150 2024-09-13T14:22:43.197 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:42] ENGINE Bus STARTED 2024-09-13T14:22:43.197 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: [13/Sep/2024:14:22:42] ENGINE Error in HTTPServer.serve 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: Traceback (most recent call last): 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._run(expiration_interval) 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: s = self.context.wrap_socket( 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: return self.sslsocket_class._create( 2024-09-13T14:22:43.198 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:22:43.199 INFO:journalctl@ceph.mon.b.smithi145.stdout: self.do_handshake() 2024-09-13T14:22:43.199 INFO:journalctl@ceph.mon.b.smithi145.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:22:43.199 INFO:journalctl@ceph.mon.b.smithi145.stdout: self._sslobj.do_handshake() 2024-09-13T14:22:43.199 INFO:journalctl@ceph.mon.b.smithi145.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:22:43.199 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:42 smithi145 ceph-mon[31057]: osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:22:43.706 INFO:tasks.rados.rados.0.smithi026.stdout:1122: done (3 left) 2024-09-13T14:22:43.706 INFO:tasks.rados.rados.0.smithi026.stdout:1123: done (2 left) 2024-09-13T14:22:43.706 INFO:tasks.rados.rados.0.smithi026.stdout:1124: done (1 left) 2024-09-13T14:22:43.706 INFO:tasks.rados.rados.0.smithi026.stdout:1125: done (0 left) 2024-09-13T14:22:43.706 INFO:tasks.rados.rados.0.smithi026.stdout:1126: snap_create 2024-09-13T14:22:43.883 INFO:tasks.rados.rados.0.smithi026.stdout:1126: done (0 left) 2024-09-13T14:22:43.883 INFO:tasks.rados.rados.0.smithi026.stdout:1127: snap_remove snap 97 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[39182]: pgmap v5: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 566 MiB/s rd, 256 MiB/s wr, 2.30k op/s; 572 B/s, 2 objects/s recovering 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[39182]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[39182]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[39182]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[97022]: pgmap v5: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 566 MiB/s rd, 256 MiB/s wr, 2.30k op/s; 572 B/s, 2 objects/s recovering 2024-09-13T14:22:44.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[97022]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:22:44.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[97022]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:22:44.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:43 smithi026 ceph-mon[97022]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:22:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:43 smithi145 ceph-mon[31057]: pgmap v5: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 566 MiB/s rd, 256 MiB/s wr, 2.30k op/s; 572 B/s, 2 objects/s recovering 2024-09-13T14:22:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:43 smithi145 ceph-mon[31057]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:22:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:43 smithi145 ceph-mon[31057]: osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:22:44.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:43 smithi145 ceph-mon[31057]: osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:22:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:1127: done (0 left) 2024-09-13T14:22:44.885 INFO:tasks.rados.rados.0.smithi026.stdout:1128: snap_remove snap 100 2024-09-13T14:22:45.886 INFO:tasks.rados.rados.0.smithi026.stdout:1128: done (0 left) 2024-09-13T14:22:45.886 INFO:tasks.rados.rados.0.smithi026.stdout:1129: delete oid 6 current snap is 106 2024-09-13T14:22:45.895 INFO:tasks.rados.rados.0.smithi026.stdout:1129: done (0 left) 2024-09-13T14:22:45.895 INFO:tasks.rados.rados.0.smithi026.stdout:1130: read oid 23 snap 105 2024-09-13T14:22:45.895 INFO:tasks.rados.rados.0.smithi026.stdout:1130: expect (ObjNum 311 snap 85 seq_num 311) 2024-09-13T14:22:45.895 INFO:tasks.rados.rados.0.smithi026.stdout:1131: delete oid 13 current snap is 106 2024-09-13T14:22:45.900 INFO:tasks.rados.rados.0.smithi026.stdout:1131: done (1 left) 2024-09-13T14:22:45.901 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 106 2024-09-13T14:22:45.901 INFO:tasks.rados.rados.0.smithi026.stdout:1132: seq_num 378 ranges {2703360=737280,3440640=720896,4161536=81920} 2024-09-13T14:22:45.920 INFO:tasks.rados.rados.0.smithi026.stdout:1132: writing smithi02683976-25 from 2703360 to 3440640 tid 1 2024-09-13T14:22:45.926 INFO:tasks.rados.rados.0.smithi026.stdout:1132: writing smithi02683976-25 from 3440640 to 4161536 tid 2 2024-09-13T14:22:45.927 INFO:tasks.rados.rados.0.smithi026.stdout:1132: writing smithi02683976-25 from 4161536 to 4243456 tid 3 2024-09-13T14:22:45.927 INFO:tasks.rados.rados.0.smithi026.stdout:1133: read oid 21 snap -1 2024-09-13T14:22:45.927 INFO:tasks.rados.rados.0.smithi026.stdout:1133: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:22:45.927 INFO:tasks.rados.rados.0.smithi026.stdout:1134: snap_remove snap 101 2024-09-13T14:22:46.114 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: pgmap v9: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 188 MiB/s rd, 85 MiB/s wr, 765 op/s; 190 B/s, 0 objects/s recovering 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.115 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:45 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: pgmap v9: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 188 MiB/s rd, 85 MiB/s wr, 765 op/s; 190 B/s, 0 objects/s recovering 2024-09-13T14:22:46.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:22:46.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: pgmap v9: 121 pgs: 121 active+clean; 165 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 188 MiB/s rd, 85 MiB/s wr, 765 op/s; 190 B/s, 0 objects/s recovering 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:45 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.900 INFO:tasks.rados.rados.0.smithi026.stdout:1134: done (3 left) 2024-09-13T14:22:46.900 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 106 2024-09-13T14:22:46.900 INFO:tasks.rados.rados.0.smithi026.stdout:1135: seq_num 379 ranges {4644864=753664,5398528=606208,6004736=196608} 2024-09-13T14:22:46.929 INFO:tasks.rados.rados.0.smithi026.stdout:1135: writing smithi02683976-22 from 4644864 to 5398528 tid 1 2024-09-13T14:22:46.932 INFO:tasks.rados.rados.0.smithi026.stdout:1135: writing smithi02683976-22 from 5398528 to 6004736 tid 2 2024-09-13T14:22:46.934 INFO:tasks.rados.rados.0.smithi026.stdout:1135: writing smithi02683976-22 from 6004736 to 6201344 tid 3 2024-09-13T14:22:46.934 INFO:tasks.rados.rados.0.smithi026.stdout:1136: rollback oid 10 current snap is 106 2024-09-13T14:22:46.934 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 102 2024-09-13T14:22:46.934 INFO:tasks.rados.rados.0.smithi026.stdout:1130: expect (ObjNum 311 snap 85 seq_num 311) 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: finishing write tid 1 to smithi02683976-25 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: finishing write tid 2 to smithi02683976-25 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: finishing write tid 3 to smithi02683976-25 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: finishing write tid 4 to smithi02683976-25 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: finishing write tid 5 to smithi02683976-25 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 360 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1132: left oid 25 (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:22:46.949 INFO:tasks.rados.rados.0.smithi026.stdout:1133: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:22:46.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-mon[31057]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:22:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:46.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:46 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: finishing write tid 1 to smithi02683976-22 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: finishing write tid 2 to smithi02683976-22 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: finishing write tid 3 to smithi02683976-22 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: finishing write tid 4 to smithi02683976-22 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: finishing write tid 5 to smithi02683976-22 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 559 (ObjNum 379 snap 106 seq_num 379) dirty exists 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1135: left oid 22 (ObjNum 379 snap 106 seq_num 379) 2024-09-13T14:22:47.119 INFO:tasks.rados.rados.0.smithi026.stdout:1136: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1136: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1136: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 327 (ObjNum 296 snap 82 seq_num 296) dirty exists 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1130: done (4 left) 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1132: done (3 left) 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1133: done (2 left) 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1135: done (1 left) 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1136: done (0 left) 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 106 2024-09-13T14:22:47.120 INFO:tasks.rados.rados.0.smithi026.stdout:1137: seq_num 380 ranges {6201344=770048,6971392=483328,7454720=319488} 2024-09-13T14:22:47.157 INFO:tasks.rados.rados.0.smithi026.stdout:1137: writing smithi02683976-22 from 6201344 to 6971392 tid 1 2024-09-13T14:22:47.159 INFO:tasks.rados.rados.0.smithi026.stdout:1137: writing smithi02683976-22 from 6971392 to 7454720 tid 2 2024-09-13T14:22:47.161 INFO:tasks.rados.rados.0.smithi026.stdout:1137: writing smithi02683976-22 from 7454720 to 7774208 tid 3 2024-09-13T14:22:47.161 INFO:tasks.rados.rados.0.smithi026.stdout:1138: snap_create 2024-09-13T14:22:47.161 INFO:tasks.rados.rados.0.smithi026.stdout:1137: finishing write tid 1 to smithi02683976-22 2024-09-13T14:22:47.161 INFO:tasks.rados.rados.0.smithi026.stdout:1137: finishing write tid 2 to smithi02683976-22 2024-09-13T14:22:47.163 INFO:tasks.rados.rados.0.smithi026.stdout:1137: finishing write tid 3 to smithi02683976-22 2024-09-13T14:22:47.163 INFO:tasks.rados.rados.0.smithi026.stdout:1137: finishing write tid 4 to smithi02683976-22 2024-09-13T14:22:47.164 INFO:tasks.rados.rados.0.smithi026.stdout:1137: finishing write tid 5 to smithi02683976-22 2024-09-13T14:22:47.164 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 563 (ObjNum 380 snap 106 seq_num 380) dirty exists 2024-09-13T14:22:47.164 INFO:tasks.rados.rados.0.smithi026.stdout:1137: left oid 22 (ObjNum 380 snap 106 seq_num 380) 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[97022]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[39182]: osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:22:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:46 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:47.905 INFO:tasks.rados.rados.0.smithi026.stdout:1137: done (1 left) 2024-09-13T14:22:47.905 INFO:tasks.rados.rados.0.smithi026.stdout:1138: done (0 left) 2024-09-13T14:22:47.906 INFO:tasks.rados.rados.0.smithi026.stdout:1139: rollback oid 29 current snap is 107 2024-09-13T14:22:47.906 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 106 2024-09-13T14:22:47.906 INFO:tasks.rados.rados.0.smithi026.stdout:1140: snap_remove snap 95 2024-09-13T14:22:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:47 smithi026 ceph-mon[97022]: pgmap v12: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 365 KiB/s wr, 21 op/s 2024-09-13T14:22:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:47 smithi026 ceph-mon[97022]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:22:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:47 smithi026 ceph-mon[39182]: pgmap v12: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 365 KiB/s wr, 21 op/s 2024-09-13T14:22:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:47 smithi026 ceph-mon[39182]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:22:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:47 smithi145 ceph-mon[31057]: pgmap v12: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 365 KiB/s wr, 21 op/s 2024-09-13T14:22:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:47 smithi145 ceph-mon[31057]: osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:22:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1140: done (1 left) 2024-09-13T14:22:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1141: setattr oid 26 current snap is 107 2024-09-13T14:22:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1139: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:22:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 0 (ObjNum 65 snap 0 seq_num 1574039275) dirty dne 2024-09-13T14:22:48.914 INFO:tasks.rados.rados.0.smithi026.stdout:1139: done (1 left) 2024-09-13T14:22:48.914 INFO:tasks.rados.rados.0.smithi026.stdout:1142: delete oid 11 current snap is 107 2024-09-13T14:22:48.919 INFO:tasks.rados.rados.0.smithi026.stdout:1142: done (1 left) 2024-09-13T14:22:48.919 INFO:tasks.rados.rados.0.smithi026.stdout:1143: snap_remove snap 105 2024-09-13T14:22:48.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:48 smithi145 ceph-mon[31057]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:22:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:48 smithi026 ceph-mon[97022]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:22:49.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:48 smithi026 ceph-mon[39182]: osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:22:49.917 INFO:tasks.rados.rados.0.smithi026.stdout:1143: done (1 left) 2024-09-13T14:22:49.917 INFO:tasks.rados.rados.0.smithi026.stdout:1144: delete oid 11 current snap is 107 2024-09-13T14:22:49.917 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 564 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:22:49.923 INFO:tasks.rados.rados.0.smithi026.stdout:1141: done (1 left) 2024-09-13T14:22:49.923 INFO:tasks.rados.rados.0.smithi026.stdout:1144: done (0 left) 2024-09-13T14:22:49.923 INFO:tasks.rados.rados.0.smithi026.stdout:1145: snap_remove snap 102 2024-09-13T14:22:50.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:49 smithi145 ceph-mon[31057]: pgmap v15: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 26 op/s 2024-09-13T14:22:50.021 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:49 smithi145 ceph-mon[31057]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:22:50.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:49 smithi026 ceph-mon[39182]: pgmap v15: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 26 op/s 2024-09-13T14:22:50.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:49 smithi026 ceph-mon[39182]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:22:50.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:49 smithi026 ceph-mon[97022]: pgmap v15: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 26 op/s 2024-09-13T14:22:50.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:49 smithi026 ceph-mon[97022]: osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:22:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1145: done (0 left) 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1146: read oid 6 snap -1 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1146: expect deleted 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1147: copy_from oid 50 from oid 30 current snap is 107 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1148: rmattr oid 49 current snap is 107 2024-09-13T14:22:50.921 INFO:tasks.rados.rados.0.smithi026.stdout:1149: delete oid 45 current snap is 107 2024-09-13T14:22:50.926 INFO:tasks.rados.rados.0.smithi026.stdout:1146: done (3 left) 2024-09-13T14:22:50.926 INFO:tasks.rados.rados.0.smithi026.stdout:1149: done (2 left) 2024-09-13T14:22:50.927 INFO:tasks.rados.rados.0.smithi026.stdout:1150: delete oid 24 current snap is 107 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 339 (ObjNum 273 snap 70 seq_num 273) dirty exists 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1148: done (2 left) 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1150: done (1 left) 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1151: read oid 39 snap 98 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1151: expect (ObjNum 318 snap 86 seq_num 318) 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1152: read oid 18 snap -1 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1152: expect (ObjNum 338 snap 91 seq_num 338) 2024-09-13T14:22:50.931 INFO:tasks.rados.rados.0.smithi026.stdout:1153: snap_create 2024-09-13T14:22:50.940 INFO:tasks.rados.rados.0.smithi026.stdout:1151: expect (ObjNum 318 snap 86 seq_num 318) 2024-09-13T14:22:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:1147: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:22:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:1147: finishing copy_from to smithi02683976-50 2024-09-13T14:22:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 297 (ObjNum 366 snap 102 seq_num 366) dirty exists 2024-09-13T14:22:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:1152: expect (ObjNum 338 snap 91 seq_num 338) 2024-09-13T14:22:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:22:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:51.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:51.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:51.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:50 smithi026 ceph-mon[97022]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:22:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.conf 2024-09-13T14:22:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.conf 2024-09-13T14:22:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:50 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:22:51.620 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:22:51] "GET /metrics HTTP/1.1" 200 33385 "" "Prometheus/2.51.0" 2024-09-13T14:22:51.927 INFO:tasks.rados.rados.0.smithi026.stdout:1147: done (3 left) 2024-09-13T14:22:51.928 INFO:tasks.rados.rados.0.smithi026.stdout:1151: done (2 left) 2024-09-13T14:22:51.928 INFO:tasks.rados.rados.0.smithi026.stdout:1152: done (1 left) 2024-09-13T14:22:51.928 INFO:tasks.rados.rados.0.smithi026.stdout:1153: done (0 left) 2024-09-13T14:22:51.928 INFO:tasks.rados.rados.0.smithi026.stdout:1154: copy_from oid 23 from oid 38 current snap is 108 2024-09-13T14:22:51.929 INFO:tasks.rados.rados.0.smithi026.stdout:1155: snap_remove snap 106 2024-09-13T14:22:52.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: pgmap v18: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-13T14:22:52.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:22:52.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.211 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:51 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: pgmap v18: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: pgmap v18: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: Updating smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: Updating smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:52.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:22:52.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:22:52.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:22:52.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:22:52.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:22:52.933 INFO:tasks.rados.rados.0.smithi026.stdout:1155: done (1 left) 2024-09-13T14:22:52.934 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 108 2024-09-13T14:22:52.934 INFO:tasks.rados.rados.0.smithi026.stdout:1156: seq_num 385 ranges {3571712=450560,4022272=761856,4784128=442368} 2024-09-13T14:22:52.955 INFO:tasks.rados.rados.0.smithi026.stdout:1156: writing smithi02683976-12 from 3571712 to 4022272 tid 1 2024-09-13T14:22:52.959 INFO:tasks.rados.rados.0.smithi026.stdout:1156: writing smithi02683976-12 from 4022272 to 4784128 tid 2 2024-09-13T14:22:52.962 INFO:tasks.rados.rados.0.smithi026.stdout:1156: writing smithi02683976-12 from 4784128 to 5226496 tid 3 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1157: copy_from oid 44 from oid 4 current snap is 108 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1154: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1154: finishing copy_from to smithi02683976-23 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 483 (ObjNum 333 snap 91 seq_num 333) dirty exists 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1156: finishing write tid 1 to smithi02683976-12 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1154: done (2 left) 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1158: rmattr oid 15 current snap is 108 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1158: done (2 left) 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 108 2024-09-13T14:22:52.963 INFO:tasks.rados.rados.0.smithi026.stdout:1159: seq_num 386 ranges {6905856=466944,7372800=638976,8011776=565248} 2024-09-13T14:22:53.003 INFO:tasks.rados.rados.0.smithi026.stdout:1159: writing smithi02683976-26 from 6905856 to 7372800 tid 1 2024-09-13T14:22:53.006 INFO:tasks.rados.rados.0.smithi026.stdout:1159: writing smithi02683976-26 from 7372800 to 8011776 tid 2 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1159: writing smithi02683976-26 from 8011776 to 8577024 tid 3 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1160: snap_create 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1156: finishing write tid 2 to smithi02683976-12 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1156: finishing write tid 3 to smithi02683976-12 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1156: finishing write tid 4 to smithi02683976-12 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1156: finishing write tid 5 to smithi02683976-12 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 568 (ObjNum 385 snap 108 seq_num 385) dirty exists 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1156: left oid 12 (ObjNum 385 snap 108 seq_num 385) 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1157: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1157: finishing copy_from to smithi02683976-44 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 290 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1159: finishing write tid 1 to smithi02683976-26 2024-09-13T14:22:53.010 INFO:tasks.rados.rados.0.smithi026.stdout:1159: finishing write tid 2 to smithi02683976-26 2024-09-13T14:22:53.013 INFO:tasks.rados.rados.0.smithi026.stdout:1159: finishing write tid 3 to smithi02683976-26 2024-09-13T14:22:53.013 INFO:tasks.rados.rados.0.smithi026.stdout:1159: finishing write tid 4 to smithi02683976-26 2024-09-13T14:22:53.014 INFO:tasks.rados.rados.0.smithi026.stdout:1159: finishing write tid 5 to smithi02683976-26 2024-09-13T14:22:53.014 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 572 (ObjNum 386 snap 108 seq_num 386) dirty exists 2024-09-13T14:22:53.014 INFO:tasks.rados.rados.0.smithi026.stdout:1159: left oid 26 (ObjNum 386 snap 108 seq_num 386) 2024-09-13T14:22:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:52 smithi145 ceph-mon[31057]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:22:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:52 smithi145 ceph-mon[31057]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:52 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:52 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:52 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[39182]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:22:53.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[39182]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:53.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:53.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[39182]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[97022]: osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[97022]: from='mgr.25142 ' entity='mgr.y' 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:22:53.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.886 INFO:tasks.rados.rados.0.smithi026.stdout:1156: done (3 left) 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1157: done (2 left) 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1159: done (1 left) 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1160: done (0 left) 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1161: rollback oid 26 current snap is 109 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 104 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1162: copy_from oid 48 from oid 14 current snap is 109 2024-09-13T14:22:53.979 INFO:tasks.rados.rados.0.smithi026.stdout:1163: snap_create 2024-09-13T14:22:53.980 INFO:tasks.rados.rados.0.smithi026.stdout:1161: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:22:53.980 INFO:tasks.rados.rados.0.smithi026.stdout:1161: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:22:53.981 INFO:tasks.rados.rados.0.smithi026.stdout:1161: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:22:53.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 575 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:22:53.981 INFO:tasks.rados.rados.0.smithi026.stdout:1162: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:22:53.981 INFO:tasks.rados.rados.0.smithi026.stdout:1162: finishing copy_from to smithi02683976-48 2024-09-13T14:22:53.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 287 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: Upgrade: Updating mon.c 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: Deploying daemon mon.c on smithi026 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: pgmap v21: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: mgrmap e42: y(active, since 12s), standbys: x 2024-09-13T14:22:54.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[39182]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: Upgrade: Updating mon.c 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: Deploying daemon mon.c on smithi026 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: pgmap v21: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: mgrmap e42: y(active, since 12s), standbys: x 2024-09-13T14:22:54.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:53 smithi026 ceph-mon[97022]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:22:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: Upgrade: Updating mon.c 2024-09-13T14:22:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: Deploying daemon mon.c on smithi026 2024-09-13T14:22:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: pgmap v21: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:54.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:22:54.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: mgrmap e42: y(active, since 12s), standbys: x 2024-09-13T14:22:54.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:53 smithi145 ceph-mon[31057]: osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:22:54.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:54 smithi026 systemd[1]: Stopping Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:54.839 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE Bus STOPPING 2024-09-13T14:22:54.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c[39159]: 2024-09-13T14:22:54.497+0000 7f63c9388640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:22:54.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c[39159]: 2024-09-13T14:22:54.497+0000 7f63c9388640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:22:54.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:54 smithi026 podman[99260]: 2024-09-13 14:22:54.549053623 +0000 UTC m=+0.192802795 container died 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 podman[99260]: 2024-09-13 14:22:55.030183929 +0000 UTC m=+0.673933143 container cleanup 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD) 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 bash[99260]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE Bus STOPPED 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE Bus STARTING 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE Serving on http://:::9283 2024-09-13T14:22:55.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:22:54 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:22:54] ENGINE Bus STARTED 2024-09-13T14:22:55.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 podman[99276]: 2024-09-13 14:22:55.273568893 +0000 UTC m=+0.722086186 container remove 671e5f478fafa4624fa42fc2384c3da2cd9ce92251b7fb0b612b81d0a87076f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T14:22:55.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:22:55.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 systemd[1]: Stopped Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:55.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service: Consumed 8.787s CPU time. 2024-09-13T14:22:55.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 systemd[1]: Starting Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:55.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 podman[99423]: 2024-09-13 14:22:55.816055441 +0000 UTC m=+0.101973644 container create f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T14:22:55.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:55 smithi026 podman[99423]: 2024-09-13 14:22:55.729988647 +0000 UTC m=+0.015906851 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 podman[99423]: 2024-09-13 14:22:56.052417987 +0000 UTC m=+0.338336191 container init f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 podman[99423]: 2024-09-13 14:22:56.057357785 +0000 UTC m=+0.343275989 container start f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: pidfile_write: ignore empty --pid-file 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: load: jerasure load: lrc 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Git sha 0 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:22:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: DB SUMMARY 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: DB Session ID: P2BUWVD512GAK0FG2Z5Q 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 11663475 ; 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.env: 0x55aa92048160 2024-09-13T14:22:56.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:22:56.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.info_log: 0x55aa938c3880 2024-09-13T14:22:56.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.statistics: (nil) 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.use_fsync: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:22:56.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.db_log_dir: 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.wal_dir: 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:22:56.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:22:56.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.write_buffer_manager: 0x55aa938c6960 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:22:56.172 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.unordered_write: 0 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.row_cache: None 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.wal_filter: None 2024-09-13T14:22:56.173 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.wal_compression: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:22:56.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:22:56.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_open_files: -1 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Compression algorithms supported: 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kZSTD supported: 0 2024-09-13T14:22:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:22:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.merge_operator: 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_filter: None 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55aa938c34e0) 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2024-09-13T14:22:56.178 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 4 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x55aa938b6850 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:22:56.179 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: bloomfilter 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:22:56.180 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 5 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_auto_readahead_size: 262144 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: prepopulate_block_cache: 0 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression: NoCompression 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:22:56.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.num_levels: 7 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:22:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:22:56.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:22:56.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:22:56.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:22:56.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:22:56.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:22:56.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.ttl: 2592000 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:22:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:22:56.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:22:56.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:22:56.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:22:56.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:22:56.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:22:56.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13472, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a0c2a6c1-eb6c-4e6d-ab83-f52a008f406e 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237376084928, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237376110790, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7239669, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13477, "largest_seqno": 17463, "table_properties": {"data_size": 7221661, "index_size": 12545, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4421, "raw_key_size": 40255, "raw_average_key_size": 23, "raw_value_size": 7184756, "raw_average_value_size": 4124, "num_data_blocks": 543, "num_entries": 1742, "num_filter_entries": 1742, "num_deletions": 315, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237376, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a0c2a6c1-eb6c-4e6d-ab83-f52a008f406e", "db_session_id": "P2BUWVD512GAK0FG2Z5Q", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237376110920, "job": 1, "event": "recovery_finished"} 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:22:56.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55aa938e6e00 2024-09-13T14:22:56.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: DB pointer 0x55aa939e6000 2024-09-13T14:22:56.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:22:56.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:22:56.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2024-09-13T14:22:56.193 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:22:56.194 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 6.90 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 714.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: L6 1/0 7.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 2/0 14.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 714.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 714.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 714.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:22:56.195 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.01 GB write, 237.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.01 GB write, 237.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache BinnedLRUCache@0x55aa938b6850#7 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:22:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: starting mon.c rank 2 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???) e3 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).mds e8 new map 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).mds e8 print_map 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: e8 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:22:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: 1 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: Filesystem 'cephfs' (1) 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: fs_name cephfs 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: epoch 8 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: created 2024-09-13T14:15:28.665431+0000 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: modified 2024-09-13T14:15:41.581255+0000 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: tableserver 0 2024-09-13T14:22:56.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: root 0 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_timeout 60 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_autoclose 300 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_file_size 1099511627776 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_xattr_size 65536 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: required_client_features {} 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure 0 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure_osd_epoch 0 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_mds 1 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: in 0 2024-09-13T14:22:56.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: up {0=24331} 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: failed 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: damaged 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: stopped 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_pools [3] 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_pool 2 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: inline_data disabled 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: balancer 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: bal_rank_mask -1 2024-09-13T14:22:56.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: standby_count_wanted 0 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.b{0:24331} state up:active seq 6 addr [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Standby daemons: 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.a{-1:14421} state up:standby seq 1 addr [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).osd e286 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).osd e286 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).osd e286 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:56.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).osd e286 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:22:56.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 ceph-mon[99461]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-13T14:22:56.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 bash[99423]: f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc 2024-09-13T14:22:56.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:56 smithi026 systemd[1]: Started Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:56.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:56.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:57.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:22:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:22:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:57.301 INFO:tasks.rados.rados.0.smithi026.stdout:1161: done (2 left) 2024-09-13T14:22:57.301 INFO:tasks.rados.rados.0.smithi026.stdout:1162: done (1 left) 2024-09-13T14:22:57.301 INFO:tasks.rados.rados.0.smithi026.stdout:1163: done (0 left) 2024-09-13T14:22:57.301 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 110 2024-09-13T14:22:57.301 INFO:tasks.rados.rados.0.smithi026.stdout:1164: seq_num 387 ranges {3121152=483328,3604480=524288,4128768=679936} 2024-09-13T14:22:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1164: writing smithi02683976-23 from 3121152 to 3604480 tid 1 2024-09-13T14:22:57.323 INFO:tasks.rados.rados.0.smithi026.stdout:1164: writing smithi02683976-23 from 3604480 to 4128768 tid 2 2024-09-13T14:22:57.328 INFO:tasks.rados.rados.0.smithi026.stdout:1164: writing smithi02683976-23 from 4128768 to 4808704 tid 3 2024-09-13T14:22:57.328 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 110 2024-09-13T14:22:57.328 INFO:tasks.rados.rados.0.smithi026.stdout:1164: finishing write tid 1 to smithi02683976-23 2024-09-13T14:22:57.328 INFO:tasks.rados.rados.0.smithi026.stdout:1165: seq_num 388 ranges {4497408=507904,5005312=458752,5464064=606208,6070272=139264} 2024-09-13T14:22:57.355 INFO:tasks.rados.rados.0.smithi026.stdout:1165: writing smithi02683976-36 from 4497408 to 5005312 tid 1 2024-09-13T14:22:57.358 INFO:tasks.rados.rados.0.smithi026.stdout:1165: writing smithi02683976-36 from 5005312 to 5464064 tid 2 2024-09-13T14:22:57.361 INFO:tasks.rados.rados.0.smithi026.stdout:1165: writing smithi02683976-36 from 5464064 to 6070272 tid 3 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:1165: writing smithi02683976-36 from 6070272 to 6209536 tid 4 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 110 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:1164: finishing write tid 2 to smithi02683976-23 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:1164: finishing write tid 3 to smithi02683976-23 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:1164: finishing write tid 4 to smithi02683976-23 2024-09-13T14:22:57.363 INFO:tasks.rados.rados.0.smithi026.stdout:1164: finishing write tid 5 to smithi02683976-23 2024-09-13T14:22:57.364 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 510 (ObjNum 387 snap 110 seq_num 387) dirty exists 2024-09-13T14:22:57.364 INFO:tasks.rados.rados.0.smithi026.stdout:1164: left oid 23 (ObjNum 387 snap 110 seq_num 387) 2024-09-13T14:22:57.364 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 1 to smithi02683976-36 2024-09-13T14:22:57.364 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 2 to smithi02683976-36 2024-09-13T14:22:57.364 INFO:tasks.rados.rados.0.smithi026.stdout:1166: seq_num 389 ranges {5226496=524288,5750784=745472,6496256=458752} 2024-09-13T14:22:57.394 INFO:tasks.rados.rados.0.smithi026.stdout:1166: writing smithi02683976-12 from 5226496 to 5750784 tid 1 2024-09-13T14:22:57.398 INFO:tasks.rados.rados.0.smithi026.stdout:1166: writing smithi02683976-12 from 5750784 to 6496256 tid 2 2024-09-13T14:22:57.401 INFO:tasks.rados.rados.0.smithi026.stdout:1166: writing smithi02683976-12 from 6496256 to 6955008 tid 3 2024-09-13T14:22:57.401 INFO:tasks.rados.rados.0.smithi026.stdout:1164: done (2 left) 2024-09-13T14:22:57.401 INFO:tasks.rados.rados.0.smithi026.stdout:1167: snap_create 2024-09-13T14:22:57.401 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 3 to smithi02683976-36 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 4 to smithi02683976-36 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 5 to smithi02683976-36 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1165: finishing write tid 6 to smithi02683976-36 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 588 (ObjNum 388 snap 110 seq_num 388) dirty exists 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1165: left oid 36 (ObjNum 388 snap 110 seq_num 388) 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1166: finishing write tid 1 to smithi02683976-12 2024-09-13T14:22:57.402 INFO:tasks.rados.rados.0.smithi026.stdout:1166: finishing write tid 2 to smithi02683976-12 2024-09-13T14:22:57.404 INFO:tasks.rados.rados.0.smithi026.stdout:1166: finishing write tid 3 to smithi02683976-12 2024-09-13T14:22:57.404 INFO:tasks.rados.rados.0.smithi026.stdout:1166: finishing write tid 4 to smithi02683976-12 2024-09-13T14:22:57.405 INFO:tasks.rados.rados.0.smithi026.stdout:1166: finishing write tid 5 to smithi02683976-12 2024-09-13T14:22:57.405 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 579 (ObjNum 389 snap 110 seq_num 389) dirty exists 2024-09-13T14:22:57.405 INFO:tasks.rados.rados.0.smithi026.stdout:1166: left oid 12 (ObjNum 389 snap 110 seq_num 389) 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: pgmap v24: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: mon.c calling monitor election 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: mon.a calling monitor election 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: monmap epoch 3 2024-09-13T14:22:57.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: min_mon_release 17 (quincy) 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: election_strategy: 1 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: mgrmap e42: y(active, since 15s), standbys: x 2024-09-13T14:22:57.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: overall HEALTH_OK 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: pgmap v24: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: mon.c calling monitor election 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: mon.a calling monitor election 2024-09-13T14:22:57.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: monmap epoch 3 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: min_mon_release 17 (quincy) 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: election_strategy: 1 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:57.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: mgrmap e42: y(active, since 15s), standbys: x 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: overall HEALTH_OK 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:57.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: pgmap v24: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.1 MiB/s wr, 25 op/s 2024-09-13T14:22:57.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: mon.c calling monitor election 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: mon.a calling monitor election 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: monmap epoch 3 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: last_changed 2024-09-13T14:11:28.329710+0000 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:22:57.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: min_mon_release 17 (quincy) 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: election_strategy: 1 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: mgrmap e42: y(active, since 15s), standbys: x 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: overall HEALTH_OK 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:22:57.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:57 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:58.297 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:22:58.309 INFO:tasks.rados.rados.0.smithi026.stdout:1165: done (2 left) 2024-09-13T14:22:58.309 INFO:tasks.rados.rados.0.smithi026.stdout:1166: done (1 left) 2024-09-13T14:22:58.309 INFO:tasks.rados.rados.0.smithi026.stdout:1167: done (0 left) 2024-09-13T14:22:58.309 INFO:tasks.rados.rados.0.smithi026.stdout:1168: read oid 16 snap -1 2024-09-13T14:22:58.309 INFO:tasks.rados.rados.0.smithi026.stdout:1168: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1169: read oid 47 snap -1 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1169: expect (ObjNum 372 snap 104 seq_num 372) 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1170: read oid 29 snap -1 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1170: expect deleted 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1171: rollback oid 14 current snap is 111 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 109 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 111 2024-09-13T14:22:58.310 INFO:tasks.rados.rados.0.smithi026.stdout:1172: seq_num 390 ranges {0=540672,540672=630784,1171456=573440} 2024-09-13T14:22:58.314 INFO:tasks.rados.rados.0.smithi026.stdout:1172: writing smithi02683976-13 from 0 to 540672 tid 1 2024-09-13T14:22:58.318 INFO:tasks.rados.rados.0.smithi026.stdout:1172: writing smithi02683976-13 from 540672 to 1171456 tid 2 2024-09-13T14:22:58.323 INFO:tasks.rados.rados.0.smithi026.stdout:1172: writing smithi02683976-13 from 1171456 to 1744896 tid 3 2024-09-13T14:22:58.323 INFO:tasks.rados.rados.0.smithi026.stdout:1168: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:22:58.366 INFO:tasks.rados.rados.0.smithi026.stdout:1169: expect (ObjNum 372 snap 104 seq_num 372) 2024-09-13T14:22:58.426 INFO:tasks.rados.rados.0.smithi026.stdout:1171: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:22:58.426 INFO:tasks.rados.rados.0.smithi026.stdout:1171: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:22:58.426 INFO:tasks.rados.rados.0.smithi026.stdout:1171: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:22:58.426 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 514 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:22:58.426 INFO:tasks.rados.rados.0.smithi026.stdout:1168: done (4 left) 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1169: done (3 left) 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1170: done (2 left) 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1171: done (1 left) 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1173: read oid 6 snap -1 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1173: expect deleted 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1174: snap_create 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: finishing write tid 1 to smithi02683976-13 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: finishing write tid 2 to smithi02683976-13 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: finishing write tid 3 to smithi02683976-13 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: finishing write tid 4 to smithi02683976-13 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: finishing write tid 5 to smithi02683976-13 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 602 (ObjNum 390 snap 111 seq_num 390) dirty exists 2024-09-13T14:22:58.427 INFO:tasks.rados.rados.0.smithi026.stdout:1172: left oid 13 (ObjNum 390 snap 111 seq_num 390) 2024-09-13T14:22:58.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:58 smithi026 ceph-mon[99461]: pgmap v26: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:58.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:58 smithi026 ceph-mon[99461]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:22:58.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:58 smithi026 ceph-mon[97022]: pgmap v26: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:58.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:58 smithi026 ceph-mon[97022]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:22:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:58 smithi145 ceph-mon[31057]: pgmap v26: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:58 smithi145 ceph-mon[31057]: osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (102s) 13s ago 6m 17.5M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (78s) 12s ago 6m 80.3M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (7m) 13s ago 7m 14.7M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (7m) 12s ago 7m 20.7M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (39s) 12s ago 11m 450M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (68s) 13s ago 12m 489M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (25s) 13s ago 13m 33.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (11m) 12s ago 11m 59.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7976afbb1510 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 starting - - - 2048M 2024-09-13T14:22:58.936 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (96s) 13s ago 6m 9428k - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (73s) 12s ago 5m 8040k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (10m) 13s ago 10m 158M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (10m) 13s ago 10m 148M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (10m) 13s ago 10m 153M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (9m) 13s ago 9m 88.5M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (9m) 12s ago 9m 164M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (8m) 12s ago 8m 148M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:22:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (8m) 12s ago 8m 100M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:22:58.938 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (8m) 12s ago 8m 115M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:22:58.938 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (44s) 12s ago 6m 41.7M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:22:59.298 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:22:59.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:22:59.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 11, 2024-09-13T14:22:59.300 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:22:59.300 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:22:59.300 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:22:59.314 INFO:tasks.rados.rados.0.smithi026.stdout:1172: done (2 left) 2024-09-13T14:22:59.314 INFO:tasks.rados.rados.0.smithi026.stdout:1173: done (1 left) 2024-09-13T14:22:59.314 INFO:tasks.rados.rados.0.smithi026.stdout:1174: done (0 left) 2024-09-13T14:22:59.314 INFO:tasks.rados.rados.0.smithi026.stdout:1175: rmattr oid 40 current snap is 112 2024-09-13T14:22:59.315 INFO:tasks.rados.rados.0.smithi026.stdout:1176: snap_remove snap 104 2024-09-13T14:22:59.607 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:22:59.607 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "3/20 daemons upgraded", 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:22:59.608 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:22:59.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[97022]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[97022]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:22:59.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3943151574' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:59.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[99461]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:59.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[99461]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:22:59.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:22:59 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3943151574' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:59 smithi145 ceph-mon[31057]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:59 smithi145 ceph-mon[31057]: osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:22:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:22:59 smithi145 ceph-mon[31057]: from='client.? 172.21.15.26:0/3943151574' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:00.320 INFO:tasks.rados.rados.0.smithi026.stdout:1176: done (1 left) 2024-09-13T14:23:00.320 INFO:tasks.rados.rados.0.smithi026.stdout:1177: snap_remove snap 108 2024-09-13T14:23:00.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: pgmap v29: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 425 B/s rd, 3.0 MiB/s wr, 3 op/s 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: pgmap v29: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 425 B/s rd, 3.0 MiB/s wr, 3 op/s 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:00.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: pgmap v29: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 425 B/s rd, 3.0 MiB/s wr, 3 op/s 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:00 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.319 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:23:01.121+0000 7f82de253640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T14:23:01.330 INFO:tasks.rados.rados.0.smithi026.stdout:1177: done (1 left) 2024-09-13T14:23:01.330 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 112 2024-09-13T14:23:01.330 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 347 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-13T14:23:01.330 INFO:tasks.rados.rados.0.smithi026.stdout:1178: seq_num 391 ranges {3293184=557056,3850240=516096,4366336=688128} 2024-09-13T14:23:01.351 INFO:tasks.rados.rados.0.smithi026.stdout:1178: writing smithi02683976-35 from 3293184 to 3850240 tid 1 2024-09-13T14:23:01.354 INFO:tasks.rados.rados.0.smithi026.stdout:1178: writing smithi02683976-35 from 3850240 to 4366336 tid 2 2024-09-13T14:23:01.360 INFO:tasks.rados.rados.0.smithi026.stdout:1178: writing smithi02683976-35 from 4366336 to 5054464 tid 3 2024-09-13T14:23:01.360 INFO:tasks.rados.rados.0.smithi026.stdout:1175: done (1 left) 2024-09-13T14:23:01.360 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 112 2024-09-13T14:23:01.360 INFO:tasks.rados.rados.0.smithi026.stdout:1179: seq_num 392 ranges {2596864=573440,3170304=802816,3973120=401408} 2024-09-13T14:23:01.377 INFO:tasks.rados.rados.0.smithi026.stdout:1179: writing smithi02683976-39 from 2596864 to 3170304 tid 1 2024-09-13T14:23:01.382 INFO:tasks.rados.rados.0.smithi026.stdout:1179: writing smithi02683976-39 from 3170304 to 3973120 tid 2 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:1179: writing smithi02683976-39 from 3973120 to 4374528 tid 3 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:1180: read oid 9 snap -1 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:1180: expect deleted 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 112 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:1178: finishing write tid 1 to smithi02683976-35 2024-09-13T14:23:01.385 INFO:tasks.rados.rados.0.smithi026.stdout:1181: seq_num 393 ranges {147456=589824,737280=679936,1417216=524288} 2024-09-13T14:23:01.390 INFO:tasks.rados.rados.0.smithi026.stdout:1181: writing smithi02683976-27 from 147456 to 737280 tid 1 2024-09-13T14:23:01.394 INFO:tasks.rados.rados.0.smithi026.stdout:1181: writing smithi02683976-27 from 737280 to 1417216 tid 2 2024-09-13T14:23:01.397 INFO:tasks.rados.rados.0.smithi026.stdout:1181: writing smithi02683976-27 from 1417216 to 1941504 tid 3 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1182: snap_create 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1178: finishing write tid 2 to smithi02683976-35 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1178: finishing write tid 3 to smithi02683976-35 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1178: finishing write tid 4 to smithi02683976-35 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1178: finishing write tid 5 to smithi02683976-35 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 597 (ObjNum 391 snap 112 seq_num 391) dirty exists 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1178: left oid 35 (ObjNum 391 snap 112 seq_num 391) 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1179: finishing write tid 1 to smithi02683976-39 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1179: finishing write tid 2 to smithi02683976-39 2024-09-13T14:23:01.398 INFO:tasks.rados.rados.0.smithi026.stdout:1179: finishing write tid 3 to smithi02683976-39 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:1179: finishing write tid 4 to smithi02683976-39 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:1179: finishing write tid 5 to smithi02683976-39 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 387 (ObjNum 392 snap 112 seq_num 392) dirty exists 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:1179: left oid 39 (ObjNum 392 snap 112 seq_num 392) 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:1181: finishing write tid 1 to smithi02683976-27 2024-09-13T14:23:01.399 INFO:tasks.rados.rados.0.smithi026.stdout:1181: finishing write tid 2 to smithi02683976-27 2024-09-13T14:23:01.400 INFO:tasks.rados.rados.0.smithi026.stdout:1181: finishing write tid 3 to smithi02683976-27 2024-09-13T14:23:01.400 INFO:tasks.rados.rados.0.smithi026.stdout:1181: finishing write tid 4 to smithi02683976-27 2024-09-13T14:23:01.401 INFO:tasks.rados.rados.0.smithi026.stdout:1181: finishing write tid 5 to smithi02683976-27 2024-09-13T14:23:01.402 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 602 (ObjNum 393 snap 112 seq_num 393) dirty exists 2024-09-13T14:23:01.402 INFO:tasks.rados.rados.0.smithi026.stdout:1181: left oid 27 (ObjNum 393 snap 112 seq_num 393) 2024-09-13T14:23:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:01] "GET /metrics HTTP/1.1" 200 36067 "" "Prometheus/2.51.0" 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[97022]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[97022]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[99461]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[99461]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:01 smithi145 ceph-mon[31057]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:01 smithi145 ceph-mon[31057]: osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:23:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:01 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:01 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:02.332 INFO:tasks.rados.rados.0.smithi026.stdout:1178: done (4 left) 2024-09-13T14:23:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:1179: done (3 left) 2024-09-13T14:23:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:1180: done (2 left) 2024-09-13T14:23:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:1181: done (1 left) 2024-09-13T14:23:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:1182: done (0 left) 2024-09-13T14:23:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:1183: snap_create 2024-09-13T14:23:02.499 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: pgmap v32: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.6 MiB/s wr, 4 op/s 2024-09-13T14:23:02.499 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:23:02.499 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: pgmap v32: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.6 MiB/s wr, 4 op/s 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: pgmap v32: 121 pgs: 121 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.6 MiB/s wr, 4 op/s 2024-09-13T14:23:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:23:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-13T14:23:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-13T14:23:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:02.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:02 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:03.342 INFO:tasks.rados.rados.0.smithi026.stdout:1183: done (0 left) 2024-09-13T14:23:03.342 INFO:tasks.rados.rados.0.smithi026.stdout:1184: delete oid 18 current snap is 114 2024-09-13T14:23:03.345 INFO:tasks.rados.rados.0.smithi026.stdout:1184: done (0 left) 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1185: read oid 32 snap -1 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1185: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1186: read oid 33 snap 113 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1186: expect (ObjNum 519038816 snap 21981 seq_num 518944912) 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1187: read oid 48 snap -1 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1187: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:23:03.346 INFO:tasks.rados.rados.0.smithi026.stdout:1188: delete oid 7 current snap is 114 2024-09-13T14:23:03.347 INFO:tasks.rados.rados.0.smithi026.stdout:1186: expect (ObjNum 519038816 snap 21981 seq_num 518944912) 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:1186: done (3 left) 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:1188: done (2 left) 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:1189: rollback oid 10 current snap is 114 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 91 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 114 2024-09-13T14:23:03.350 INFO:tasks.rados.rados.0.smithi026.stdout:1190: seq_num 396 ranges {1310720=638976,1949696=786432,2736128=417792} 2024-09-13T14:23:03.362 INFO:tasks.rados.rados.0.smithi026.stdout:1190: writing smithi02683976-8 from 1310720 to 1949696 tid 1 2024-09-13T14:23:03.366 INFO:tasks.rados.rados.0.smithi026.stdout:1190: writing smithi02683976-8 from 1949696 to 2736128 tid 2 2024-09-13T14:23:03.369 INFO:tasks.rados.rados.0.smithi026.stdout:1190: writing smithi02683976-8 from 2736128 to 3153920 tid 3 2024-09-13T14:23:03.369 INFO:tasks.rados.rados.0.smithi026.stdout:1191: rollback oid 24 current snap is 114 2024-09-13T14:23:03.369 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 112 2024-09-13T14:23:03.369 INFO:tasks.rados.rados.0.smithi026.stdout:1185: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:23:03.409 INFO:tasks.rados.rados.0.smithi026.stdout:1189: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:23:03.410 INFO:tasks.rados.rados.0.smithi026.stdout:1189: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:23:03.410 INFO:tasks.rados.rados.0.smithi026.stdout:1189: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:23:03.410 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 359 (ObjNum 296 snap 82 seq_num 296) dirty exists 2024-09-13T14:23:03.410 INFO:tasks.rados.rados.0.smithi026.stdout:1187: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:23:03.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:03 smithi026 ceph-mon[97022]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:23:03.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:03 smithi026 ceph-mon[99461]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:23:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:03 smithi145 ceph-mon[31057]: osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:23:03.747 INFO:tasks.rados.rados.0.smithi026.stdout:1190: finishing write tid 1 to smithi02683976-8 2024-09-13T14:23:03.747 INFO:tasks.rados.rados.0.smithi026.stdout:1190: finishing write tid 2 to smithi02683976-8 2024-09-13T14:23:03.747 INFO:tasks.rados.rados.0.smithi026.stdout:1191: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 376 (ObjNum 177 snap 0 seq_num 519563328) dirty dne 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:1190: finishing write tid 3 to smithi02683976-8 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:1185: done (4 left) 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:1187: done (3 left) 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:1189: done (2 left) 2024-09-13T14:23:03.748 INFO:tasks.rados.rados.0.smithi026.stdout:1191: done (1 left) 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:1192: delete oid 36 current snap is 114 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:1190: finishing write tid 4 to smithi02683976-8 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:1190: finishing write tid 5 to smithi02683976-8 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 210 (ObjNum 396 snap 114 seq_num 396) dirty exists 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:1190: left oid 8 (ObjNum 396 snap 114 seq_num 396) 2024-09-13T14:23:03.749 INFO:tasks.rados.rados.0.smithi026.stdout:1190: done (1 left) 2024-09-13T14:23:03.750 INFO:tasks.rados.rados.0.smithi026.stdout:1192: done (0 left) 2024-09-13T14:23:03.750 INFO:tasks.rados.rados.0.smithi026.stdout:1193: delete oid 13 current snap is 114 2024-09-13T14:23:03.751 INFO:tasks.rados.rados.0.smithi026.stdout:1193: done (0 left) 2024-09-13T14:23:03.751 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 114 2024-09-13T14:23:03.751 INFO:tasks.rados.rados.0.smithi026.stdout:1194: seq_num 399 ranges {4497408=688128,5185536=434176,5619712=745472,6365184=24576} 2024-09-13T14:23:03.779 INFO:tasks.rados.rados.0.smithi026.stdout:1194: writing smithi02683976-10 from 4497408 to 5185536 tid 1 2024-09-13T14:23:03.782 INFO:tasks.rados.rados.0.smithi026.stdout:1194: writing smithi02683976-10 from 5185536 to 5619712 tid 2 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1194: writing smithi02683976-10 from 5619712 to 6365184 tid 3 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1194: writing smithi02683976-10 from 6365184 to 6389760 tid 4 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1195: rmattr oid 1 current snap is 114 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1195: done (1 left) 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1196: read oid 7 snap 111 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1196: expect deleted 2024-09-13T14:23:03.787 INFO:tasks.rados.rados.0.smithi026.stdout:1197: snap_remove snap 99 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 1 to smithi02683976-10 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 2 to smithi02683976-10 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 3 to smithi02683976-10 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 4 to smithi02683976-10 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 5 to smithi02683976-10 2024-09-13T14:23:03.890 INFO:tasks.rados.rados.0.smithi026.stdout:1194: finishing write tid 6 to smithi02683976-10 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 364 (ObjNum 399 snap 114 seq_num 399) dirty exists 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1194: left oid 10 (ObjNum 399 snap 114 seq_num 399) 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1194: done (2 left) 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1196: done (1 left) 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1197: done (0 left) 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1198: read oid 16 snap -1 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1198: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:23:03.891 INFO:tasks.rados.rados.0.smithi026.stdout:1199: snap_remove snap 91 2024-09-13T14:23:04.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: pgmap v35: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:04.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:04.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:23:04.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:04 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:23:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: pgmap v35: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:23:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:23:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: pgmap v35: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:04.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:04.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:23:04.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:23:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:1199: done (1 left) 2024-09-13T14:23:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:1200: rollback oid 49 current snap is 114 2024-09-13T14:23:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 49 to 110 2024-09-13T14:23:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:1201: snap_create 2024-09-13T14:23:04.891 INFO:tasks.rados.rados.0.smithi026.stdout:1198: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:23:04.931 INFO:tasks.rados.rados.0.smithi026.stdout:1200: finishing rollback tid 0 to smithi02683976-49 2024-09-13T14:23:04.931 INFO:tasks.rados.rados.0.smithi026.stdout:1200: finishing rollback tid 1 to smithi02683976-49 2024-09-13T14:23:04.931 INFO:tasks.rados.rados.0.smithi026.stdout:1200: finishing rollback tid 2 to smithi02683976-49 2024-09-13T14:23:04.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 374 (ObjNum 273 snap 70 seq_num 273) dirty exists 2024-09-13T14:23:05.440 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:05 smithi145 ceph-mon[31057]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:23:05.440 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:05 smithi145 ceph-mon[31057]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:23:05.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:05 smithi026 ceph-mon[97022]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:23:05.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:05 smithi026 ceph-mon[97022]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:23:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:05 smithi026 ceph-mon[99461]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:23:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:05 smithi026 ceph-mon[99461]: osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:23:05.892 INFO:tasks.rados.rados.0.smithi026.stdout:1198: done (2 left) 2024-09-13T14:23:05.892 INFO:tasks.rados.rados.0.smithi026.stdout:1200: done (1 left) 2024-09-13T14:23:05.892 INFO:tasks.rados.rados.0.smithi026.stdout:1201: done (0 left) 2024-09-13T14:23:05.892 INFO:tasks.rados.rados.0.smithi026.stdout:1202: snap_create 2024-09-13T14:23:06.011 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:06.011 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: pgmap v38: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:06.623 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-mon[31057]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:23:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: pgmap v38: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[97022]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: pgmap v38: 121 pgs: 121 active+clean; 195 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 24 op/s 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:06.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:06 smithi026 ceph-mon[99461]: osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:23:06.893 INFO:tasks.rados.rados.0.smithi026.stdout:1202: done (0 left) 2024-09-13T14:23:06.893 INFO:tasks.rados.rados.0.smithi026.stdout:1203: delete oid 39 current snap is 116 2024-09-13T14:23:06.898 INFO:tasks.rados.rados.0.smithi026.stdout:1203: done (0 left) 2024-09-13T14:23:06.899 INFO:tasks.rados.rados.0.smithi026.stdout:1204: rollback oid 33 current snap is 116 2024-09-13T14:23:06.899 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 107 2024-09-13T14:23:06.899 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 116 2024-09-13T14:23:06.899 INFO:tasks.rados.rados.0.smithi026.stdout:1205: seq_num 401 ranges {6955008=720896,7675904=606208,8282112=598016} 2024-09-13T14:23:06.941 INFO:tasks.rados.rados.0.smithi026.stdout:1205: writing smithi02683976-12 from 6955008 to 7675904 tid 1 2024-09-13T14:23:06.944 INFO:tasks.rados.rados.0.smithi026.stdout:1205: writing smithi02683976-12 from 7675904 to 8282112 tid 2 2024-09-13T14:23:06.947 INFO:tasks.rados.rados.0.smithi026.stdout:1205: writing smithi02683976-12 from 8282112 to 8880128 tid 3 2024-09-13T14:23:06.947 INFO:tasks.rados.rados.0.smithi026.stdout:1206: rollback oid 42 current snap is 116 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 98 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1204: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1204: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1204: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 628 (ObjNum 65 snap 0 seq_num 1448045166) dirty exists 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1205: finishing write tid 1 to smithi02683976-12 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1205: finishing write tid 2 to smithi02683976-12 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1204: done (2 left) 2024-09-13T14:23:06.948 INFO:tasks.rados.rados.0.smithi026.stdout:1207: snap_remove snap 107 2024-09-13T14:23:07.199 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 systemd[1]: Stopping Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 ceph-mon[31057]: Upgrade: Updating mon.b 2024-09-13T14:23:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 ceph-mon[31057]: Deploying daemon mon.b on smithi145 2024-09-13T14:23:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 ceph-mon[31057]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:23:07.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[99461]: Upgrade: Updating mon.b 2024-09-13T14:23:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[99461]: Deploying daemon mon.b on smithi145 2024-09-13T14:23:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[99461]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:23:07.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[97022]: Upgrade: Updating mon.b 2024-09-13T14:23:07.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[97022]: Deploying daemon mon.b on smithi145 2024-09-13T14:23:07.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:07 smithi026 ceph-mon[97022]: osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:23:07.803 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b[31034]: 2024-09-13T14:23:07.519+0000 7f9c72da9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:23:07.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b[31034]: 2024-09-13T14:23:07.519+0000 7f9c72da9640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:23:07.804 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 podman[88956]: 2024-09-13 14:23:07.541013842 +0000 UTC m=+0.172730893 container died 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS) 2024-09-13T14:23:08.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 podman[88956]: 2024-09-13 14:23:07.901987148 +0000 UTC m=+0.533704243 container cleanup 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa) 2024-09-13T14:23:08.178 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:07 smithi145 bash[88956]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:23:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 podman[88975]: 2024-09-13 14:23:08.178016442 +0000 UTC m=+0.632851933 container remove 7976afbb1510c3e3e492a48822c6b8203ff91d2829adbb739d380b5ecb8b33db (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:23:08.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T14:23:08.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 systemd[1]: Stopped Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:08.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.b.service: Consumed 9.945s CPU time. 2024-09-13T14:23:08.881 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 systemd[1]: Starting Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:08.881 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 podman[89107]: 2024-09-13 14:23:08.631743421 +0000 UTC m=+0.016086707 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:08.881 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 podman[89107]: 2024-09-13 14:23:08.763052588 +0000 UTC m=+0.147395906 container create 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:23:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:1207: done (2 left) 2024-09-13T14:23:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1208: read oid 31 snap -1 2024-09-13T14:23:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1208: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:23:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 116 2024-09-13T14:23:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1206: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:23:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1206: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 300 (ObjNum 228 snap 55 seq_num 228) dirty exists 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:1205: finishing write tid 3 to smithi02683976-12 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:1205: finishing write tid 4 to smithi02683976-12 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:1205: finishing write tid 5 to smithi02683976-12 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 583 (ObjNum 401 snap 116 seq_num 401) dirty exists 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:1205: left oid 12 (ObjNum 401 snap 116 seq_num 401) 2024-09-13T14:23:09.312 INFO:tasks.rados.rados.0.smithi026.stdout:1209: seq_num 402 ranges {0=737280,737280=491520,1228800=516096,1744896=196608} 2024-09-13T14:23:09.315 INFO:tasks.rados.rados.0.smithi026.stdout:1209: writing smithi02683976-29 from 0 to 737280 tid 1 2024-09-13T14:23:09.319 INFO:tasks.rados.rados.0.smithi026.stdout:1209: writing smithi02683976-29 from 737280 to 1228800 tid 2 2024-09-13T14:23:09.322 INFO:tasks.rados.rados.0.smithi026.stdout:1209: writing smithi02683976-29 from 1228800 to 1744896 tid 3 2024-09-13T14:23:09.324 INFO:tasks.rados.rados.0.smithi026.stdout:1209: writing smithi02683976-29 from 1744896 to 1941504 tid 4 2024-09-13T14:23:09.324 INFO:tasks.rados.rados.0.smithi026.stdout:1205: done (3 left) 2024-09-13T14:23:09.325 INFO:tasks.rados.rados.0.smithi026.stdout:1206: done (2 left) 2024-09-13T14:23:09.325 INFO:tasks.rados.rados.0.smithi026.stdout:1210: read oid 16 snap -1 2024-09-13T14:23:09.325 INFO:tasks.rados.rados.0.smithi026.stdout:1210: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:23:09.325 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 116 2024-09-13T14:23:09.325 INFO:tasks.rados.rados.0.smithi026.stdout:1211: seq_num 403 ranges {0=753664,753664=425984,1179648=720896,1900544=57344} 2024-09-13T14:23:09.329 INFO:tasks.rados.rados.0.smithi026.stdout:1211: writing smithi02683976-36 from 0 to 753664 tid 1 2024-09-13T14:23:09.332 INFO:tasks.rados.rados.0.smithi026.stdout:1211: writing smithi02683976-36 from 753664 to 1179648 tid 2 2024-09-13T14:23:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1211: writing smithi02683976-36 from 1179648 to 1900544 tid 3 2024-09-13T14:23:09.338 INFO:tasks.rados.rados.0.smithi026.stdout:1211: writing smithi02683976-36 from 1900544 to 1957888 tid 4 2024-09-13T14:23:09.338 INFO:tasks.rados.rados.0.smithi026.stdout:1212: delete oid 1 current snap is 116 2024-09-13T14:23:09.338 INFO:tasks.rados.rados.0.smithi026.stdout:1208: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:23:09.389 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 1 to smithi02683976-29 2024-09-13T14:23:09.389 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 2 to smithi02683976-29 2024-09-13T14:23:09.389 INFO:tasks.rados.rados.0.smithi026.stdout:1208: done (4 left) 2024-09-13T14:23:09.389 INFO:tasks.rados.rados.0.smithi026.stdout:1212: done (3 left) 2024-09-13T14:23:09.389 INFO:tasks.rados.rados.0.smithi026.stdout:1213: snap_remove snap 115 2024-09-13T14:23:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:08 smithi145 podman[89107]: 2024-09-13 14:23:08.998833285 +0000 UTC m=+0.383176561 container init 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 podman[89107]: 2024-09-13 14:23:09.003273146 +0000 UTC m=+0.387616423 container start 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: pidfile_write: ignore empty --pid-file 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: load: jerasure load: lrc 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:23:09.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Git sha 0 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: DB SUMMARY 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: DB Session ID: D5B63DM0QT11IW1H3VUN 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 12559558 ; 2024-09-13T14:23:09.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.env: 0x55f888c1e160 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.info_log: 0x55f889525880 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.statistics: (nil) 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.use_fsync: 0 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:23:09.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.db_log_dir: 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.wal_dir: 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:23:09.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.write_buffer_manager: 0x55f889528960 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:23:09.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.unordered_write: 0 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.row_cache: None 2024-09-13T14:23:09.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.wal_filter: None 2024-09-13T14:23:09.462 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:23:09.462 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:23:09.462 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.wal_compression: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:23:09.463 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_open_files: -1 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:23:09.464 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Compression algorithms supported: 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kZSTD supported: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:23:09.465 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.merge_operator: 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_filter: None 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f8895254e0) 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_type: 0 2024-09-13T14:23:09.466 INFO:journalctl@ceph.mon.b.smithi145.stdout: data_block_index_type: 0 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_shortening: 1 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: checksum: 4 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: no_block_cache: 0 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache: 0x55f889518850 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_options: 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: capacity : 536870912 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: num_shard_bits : 4 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: strict_capacity_limit : 0 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_cache_compressed: (nil) 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: persistent_cache: (nil) 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_size: 4096 2024-09-13T14:23:09.467 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_size_deviation: 10 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_restart_interval: 16 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: index_block_restart_interval: 1 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: metadata_block_size: 4096 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: partition_filters: 0 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: use_delta_encoding: 1 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: filter_policy: bloomfilter 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: whole_key_filtering: 1 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: verify_compression: 0 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: format_version: 5 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: enable_index_compression: 1 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: block_align: 0 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: max_auto_readahead_size: 262144 2024-09-13T14:23:09.468 INFO:journalctl@ceph.mon.b.smithi145.stdout: prepopulate_block_cache: 0 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression: NoCompression 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.num_levels: 7 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:23:09.469 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:23:09.470 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:23:09.471 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:23:09.472 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:23:09.473 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.ttl: 2592000 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:23:09.474 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13546, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 43575f1d-eea2-4254-9118-030aecac2e0f 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237389029875, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237389057957, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7771080, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13551, "largest_seqno": 17978, "table_properties": {"data_size": 7751308, "index_size": 13925, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4805, "raw_key_size": 44341, "raw_average_key_size": 23, "raw_value_size": 7710612, "raw_average_value_size": 4024, "num_data_blocks": 602, "num_entries": 1916, "num_filter_entries": 1916, "num_deletions": 317, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237389, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "43575f1d-eea2-4254-9118-030aecac2e0f", "db_session_id": "D5B63DM0QT11IW1H3VUN", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:23:09.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237389058088, "job": 1, "event": "recovery_finished"} 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f889548e00 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: DB pointer 0x55f889648000 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** DB Stats ** 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:23:09.476 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: L0 1/0 7.41 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 647.8 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: L6 1/0 7.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: Sum 2/0 15.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 647.8 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 647.8 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** Compaction Stats [default] ** 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 647.8 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.477 INFO:journalctl@ceph.mon.b.smithi145.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Cumulative compaction: 0.01 GB write, 225.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Interval compaction: 0.01 GB write, 225.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Block cache BinnedLRUCache@0x55f889518850#7 capacity: 512.00 MB usage: 21.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-09-13T14:23:09.478 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: starting mon.b rank 1 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???) e3 preinit fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).mds e8 new map 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).mds e8 print_map 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: e8 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: legacy client fscid: 1 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: Filesystem 'cephfs' (1) 2024-09-13T14:23:09.479 INFO:journalctl@ceph.mon.b.smithi145.stdout: fs_name cephfs 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: epoch 8 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: created 2024-09-13T14:15:28.665431+0000 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: modified 2024-09-13T14:15:41.581255+0000 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: tableserver 0 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: root 0 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: session_timeout 60 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: session_autoclose 300 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: max_file_size 1099511627776 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: max_xattr_size 65536 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: required_client_features {} 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: last_failure 0 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: last_failure_osd_epoch 0 2024-09-13T14:23:09.480 INFO:journalctl@ceph.mon.b.smithi145.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: max_mds 1 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: in 0 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: up {0=24331} 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: failed 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: damaged 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: stopped 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: data_pools [3] 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: metadata_pool 2 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: inline_data disabled 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: balancer 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: bal_rank_mask -1 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: standby_count_wanted 0 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:23:09.481 INFO:journalctl@ceph.mon.b.smithi145.stdout: [mds.b{0:24331} state up:active seq 6 addr [v2:172.21.15.145:6832/2983197144,v1:172.21.15.145:6833/2983197144] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout: Standby daemons: 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout: 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout: [mds.a{-1:14421} state up:standby seq 1 addr [v2:172.21.15.26:6834/248064250,v1:172.21.15.26:6835/248064250] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).osd e298 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).osd e298 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).osd e298 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).osd e298 crush map has features 432629308056666112, adjusting msgr requires 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 ceph-mon[89145]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 bash[89107]: 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 2024-09-13T14:23:09.482 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:09 smithi145 systemd[1]: Started Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:10.348 INFO:tasks.rados.rados.0.smithi026.stdout:1213: done (3 left) 2024-09-13T14:23:10.348 INFO:tasks.rados.rados.0.smithi026.stdout:1214: copy_from oid 9 from oid 11 current snap is 116 2024-09-13T14:23:10.348 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 3 to smithi02683976-29 2024-09-13T14:23:10.348 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 4 to smithi02683976-29 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 5 to smithi02683976-29 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:1209: finishing write tid 6 to smithi02683976-29 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 621 (ObjNum 402 snap 116 seq_num 402) dirty exists 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:1209: left oid 29 (ObjNum 402 snap 116 seq_num 402) 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 1 to smithi02683976-36 2024-09-13T14:23:10.349 INFO:tasks.rados.rados.0.smithi026.stdout:1210: expect (ObjNum 353 snap 97 seq_num 353) 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 2 to smithi02683976-36 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 3 to smithi02683976-36 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 4 to smithi02683976-36 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 5 to smithi02683976-36 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:1211: finishing write tid 6 to smithi02683976-36 2024-09-13T14:23:10.378 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 641 (ObjNum 403 snap 116 seq_num 403) dirty exists 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1211: left oid 36 (ObjNum 403 snap 116 seq_num 403) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1214: finishing copy_from to smithi02683976-9 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1214: got expected ENOENT (src dne) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1214: finishing copy_from racing read to smithi02683976-9 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1209: done (3 left) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1210: done (2 left) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1211: done (1 left) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1214: done (0 left) 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 116 2024-09-13T14:23:10.379 INFO:tasks.rados.rados.0.smithi026.stdout:1215: seq_num 405 ranges {1122304=786432,1908736=589824,2498560=614400} 2024-09-13T14:23:10.388 INFO:tasks.rados.rados.0.smithi026.stdout:1215: writing smithi02683976-16 from 1122304 to 1908736 tid 1 2024-09-13T14:23:10.392 INFO:tasks.rados.rados.0.smithi026.stdout:1215: writing smithi02683976-16 from 1908736 to 2498560 tid 2 2024-09-13T14:23:10.396 INFO:tasks.rados.rados.0.smithi026.stdout:1215: writing smithi02683976-16 from 2498560 to 3112960 tid 3 2024-09-13T14:23:10.396 INFO:tasks.rados.rados.0.smithi026.stdout:1216: delete oid 48 current snap is 116 2024-09-13T14:23:10.396 INFO:tasks.rados.rados.0.smithi026.stdout:1215: finishing write tid 1 to smithi02683976-16 2024-09-13T14:23:10.397 INFO:tasks.rados.rados.0.smithi026.stdout:1215: finishing write tid 2 to smithi02683976-16 2024-09-13T14:23:10.401 INFO:tasks.rados.rados.0.smithi026.stdout:1215: finishing write tid 3 to smithi02683976-16 2024-09-13T14:23:10.402 INFO:tasks.rados.rados.0.smithi026.stdout:1215: finishing write tid 4 to smithi02683976-16 2024-09-13T14:23:10.402 INFO:tasks.rados.rados.0.smithi026.stdout:1216: done (1 left) 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:1217: read oid 7 snap 113 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:1217: expect deleted 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:1218: rollback oid 19 current snap is 116 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 98 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 116 2024-09-13T14:23:10.403 INFO:tasks.rados.rados.0.smithi026.stdout:1219: seq_num 407 ranges {802816=425984,1228800=761856,1990656=704512,2695168=139264} 2024-09-13T14:23:10.409 INFO:tasks.rados.rados.0.smithi026.stdout:1219: writing smithi02683976-41 from 802816 to 1228800 tid 1 2024-09-13T14:23:10.413 INFO:tasks.rados.rados.0.smithi026.stdout:1219: writing smithi02683976-41 from 1228800 to 1990656 tid 2 2024-09-13T14:23:10.417 INFO:tasks.rados.rados.0.smithi026.stdout:1219: writing smithi02683976-41 from 1990656 to 2695168 tid 3 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1219: writing smithi02683976-41 from 2695168 to 2834432 tid 4 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1220: read oid 32 snap -1 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1220: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1221: rollback oid 17 current snap is 116 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 109 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1217: done (5 left) 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1222: rollback oid 44 current snap is 116 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 112 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1215: finishing write tid 5 to smithi02683976-16 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 626 (ObjNum 405 snap 116 seq_num 405) dirty exists 2024-09-13T14:23:10.419 INFO:tasks.rados.rados.0.smithi026.stdout:1215: left oid 16 (ObjNum 405 snap 116 seq_num 405) 2024-09-13T14:23:10.420 INFO:tasks.rados.rados.0.smithi026.stdout:1218: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:23:10.420 INFO:tasks.rados.rados.0.smithi026.stdout:1215: done (5 left) 2024-09-13T14:23:10.420 INFO:tasks.rados.rados.0.smithi026.stdout:1223: snap_remove snap 110 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: pgmap v43: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 757 KiB/s wr, 7 op/s 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: mon.a calling monitor election 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: mon.b calling monitor election 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: mon.c calling monitor election 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: monmap epoch 4 2024-09-13T14:23:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: last_changed 2024-09-13T14:23:09.303177+0000 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: min_mon_release 19 (squid) 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: election_strategy: 1 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:23:10.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: mgrmap e42: y(active, since 28s), standbys: x 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: overall HEALTH_OK 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: pgmap v43: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 757 KiB/s wr, 7 op/s 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: mon.a calling monitor election 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: mon.b calling monitor election 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: mon.c calling monitor election 2024-09-13T14:23:10.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: monmap epoch 4 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: last_changed 2024-09-13T14:23:09.303177+0000 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: min_mon_release 19 (squid) 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: election_strategy: 1 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:23:10.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:23:10.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: mgrmap e42: y(active, since 28s), standbys: x 2024-09-13T14:23:10.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: overall HEALTH_OK 2024-09-13T14:23:10.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:10.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: pgmap v43: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 757 KiB/s wr, 7 op/s 2024-09-13T14:23:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:23:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: mon.a calling monitor election 2024-09-13T14:23:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: mon.b calling monitor election 2024-09-13T14:23:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: mon.c calling monitor election 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: monmap epoch 4 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: fsid ad455616-71d9-11ef-bceb-c7b262605968 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: last_changed 2024-09-13T14:23:09.303177+0000 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: created 2024-09-13T14:09:50.810274+0000 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: min_mon_release 19 (squid) 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: election_strategy: 1 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: 1: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.b 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: 2: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-09-13T14:23:10.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: mgrmap e42: y(active, since 28s), standbys: x 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: overall HEALTH_OK 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:10.707 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:11.354 INFO:tasks.rados.rados.0.smithi026.stdout:1223: done (5 left) 2024-09-13T14:23:11.354 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 116 2024-09-13T14:23:11.354 INFO:tasks.rados.rados.0.smithi026.stdout:1218: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:23:11.354 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 629 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-13T14:23:11.355 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 1 to smithi02683976-41 2024-09-13T14:23:11.355 INFO:tasks.rados.rados.0.smithi026.stdout:1224: seq_num 408 ranges {0=442368,442368=647168,1089536=507904,1597440=450560} 2024-09-13T14:23:11.356 INFO:tasks.rados.rados.0.smithi026.stdout:1224: writing smithi02683976-39 from 0 to 442368 tid 1 2024-09-13T14:23:11.360 INFO:tasks.rados.rados.0.smithi026.stdout:1224: writing smithi02683976-39 from 442368 to 1089536 tid 2 2024-09-13T14:23:11.364 INFO:tasks.rados.rados.0.smithi026.stdout:1224: writing smithi02683976-39 from 1089536 to 1597440 tid 3 2024-09-13T14:23:11.367 INFO:tasks.rados.rados.0.smithi026.stdout:1224: writing smithi02683976-39 from 1597440 to 2048000 tid 4 2024-09-13T14:23:11.367 INFO:tasks.rados.rados.0.smithi026.stdout:1218: done (5 left) 2024-09-13T14:23:11.367 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 116 2024-09-13T14:23:11.368 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 2 to smithi02683976-41 2024-09-13T14:23:11.368 INFO:tasks.rados.rados.0.smithi026.stdout:1222: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:23:11.368 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 3 to smithi02683976-41 2024-09-13T14:23:11.368 INFO:tasks.rados.rados.0.smithi026.stdout:1225: seq_num 409 ranges {5054464=458752,5513216=532480,6045696=720896,6766592=352256} 2024-09-13T14:23:11.398 INFO:tasks.rados.rados.0.smithi026.stdout:1225: writing smithi02683976-35 from 5054464 to 5513216 tid 1 2024-09-13T14:23:11.401 INFO:tasks.rados.rados.0.smithi026.stdout:1225: writing smithi02683976-35 from 5513216 to 6045696 tid 2 2024-09-13T14:23:11.405 INFO:tasks.rados.rados.0.smithi026.stdout:1225: writing smithi02683976-35 from 6045696 to 6766592 tid 3 2024-09-13T14:23:11.408 INFO:tasks.rados.rados.0.smithi026.stdout:1225: writing smithi02683976-35 from 6766592 to 7118848 tid 4 2024-09-13T14:23:11.408 INFO:tasks.rados.rados.0.smithi026.stdout:1222: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:23:11.408 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 4 to smithi02683976-41 2024-09-13T14:23:11.408 INFO:tasks.rados.rados.0.smithi026.stdout:1222: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:23:11.408 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 430 (ObjNum 336 snap 91 seq_num 336) dirty exists 2024-09-13T14:23:11.409 INFO:tasks.rados.rados.0.smithi026.stdout:1220: expect (ObjNum 228 snap 55 seq_num 228) 2024-09-13T14:23:11.451 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 5 to smithi02683976-41 2024-09-13T14:23:11.451 INFO:tasks.rados.rados.0.smithi026.stdout:1221: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:23:11.451 INFO:tasks.rados.rados.0.smithi026.stdout:1221: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:23:11.451 INFO:tasks.rados.rados.0.smithi026.stdout:1219: finishing write tid 6 to smithi02683976-41 2024-09-13T14:23:11.451 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 392 (ObjNum 407 snap 116 seq_num 407) dirty exists 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1219: left oid 41 (ObjNum 407 snap 116 seq_num 407) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1221: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 395 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 1 to smithi02683976-39 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1219: done (5 left) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1220: done (4 left) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1221: done (3 left) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1222: done (2 left) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1226: read oid 28 snap -1 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1226: expect (ObjNum 307 snap 83 seq_num 307) 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1227: copy_from oid 18 from oid 40 current snap is 116 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 2 to smithi02683976-39 2024-09-13T14:23:11.452 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 3 to smithi02683976-39 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 4 to smithi02683976-39 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 5 to smithi02683976-39 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1224: finishing write tid 6 to smithi02683976-39 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 413 (ObjNum 408 snap 116 seq_num 408) dirty exists 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1224: left oid 39 (ObjNum 408 snap 116 seq_num 408) 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 1 to smithi02683976-35 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 2 to smithi02683976-35 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 3 to smithi02683976-35 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 4 to smithi02683976-35 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 5 to smithi02683976-35 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1224: done (3 left) 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1228: read oid 44 snap -1 2024-09-13T14:23:11.453 INFO:tasks.rados.rados.0.smithi026.stdout:1228: expect (ObjNum 336 snap 91 seq_num 336) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1229: rollback oid 47 current snap is 116 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1225: finishing write tid 6 to smithi02683976-35 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 657 (ObjNum 409 snap 116 seq_num 409) dirty exists 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1225: left oid 35 (ObjNum 409 snap 116 seq_num 409) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 112 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1225: done (4 left) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1230: rmattr oid 41 current snap is 116 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1230: done (4 left) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1231: read oid 20 snap -1 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1231: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1232: read oid 41 snap -1 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1232: expect (ObjNum 407 snap 116 seq_num 407) 2024-09-13T14:23:11.454 INFO:tasks.rados.rados.0.smithi026.stdout:1233: read oid 26 snap -1 2024-09-13T14:23:11.455 INFO:tasks.rados.rados.0.smithi026.stdout:1233: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:23:11.455 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 116 2024-09-13T14:23:11.455 INFO:tasks.rados.rados.0.smithi026.stdout:1234: seq_num 410 ranges {1335296=475136,1810432=417792,2228224=475136,2703360=425984,3129344=286720} 2024-09-13T14:23:11.460 INFO:tasks.rados.rados.0.smithi026.stdout:1234: writing smithi02683976-50 from 1335296 to 1810432 tid 1 2024-09-13T14:23:11.463 INFO:tasks.rados.rados.0.smithi026.stdout:1234: writing smithi02683976-50 from 1810432 to 2228224 tid 2 2024-09-13T14:23:11.466 INFO:tasks.rados.rados.0.smithi026.stdout:1234: writing smithi02683976-50 from 2228224 to 2703360 tid 3 2024-09-13T14:23:11.469 INFO:tasks.rados.rados.0.smithi026.stdout:1234: writing smithi02683976-50 from 2703360 to 3129344 tid 4 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1234: writing smithi02683976-50 from 3129344 to 3416064 tid 5 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1235: read oid 33 snap -1 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1235: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1236: read oid 39 snap -1 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1236: expect (ObjNum 408 snap 116 seq_num 408) 2024-09-13T14:23:11.471 INFO:tasks.rados.rados.0.smithi026.stdout:1237: snap_remove snap 113 2024-09-13T14:23:11.650 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:11 smithi145 ceph-mon[89145]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:23:11.650 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:11 smithi026 ceph-mon[97022]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:23:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:11 smithi026 ceph-mon[99461]: osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:23:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:11.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:11] "GET /metrics HTTP/1.1" 200 36058 "" "Prometheus/2.51.0" 2024-09-13T14:23:12.357 INFO:tasks.rados.rados.0.smithi026.stdout:1237: done (10 left) 2024-09-13T14:23:12.357 INFO:tasks.rados.rados.0.smithi026.stdout:1238: read oid 3 snap -1 2024-09-13T14:23:12.357 INFO:tasks.rados.rados.0.smithi026.stdout:1238: expect deleted 2024-09-13T14:23:12.358 INFO:tasks.rados.rados.0.smithi026.stdout:1239: read oid 19 snap 114 2024-09-13T14:23:12.358 INFO:tasks.rados.rados.0.smithi026.stdout:1239: expect (ObjNum 356 snap 99 seq_num 356) 2024-09-13T14:23:12.358 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 116 2024-09-13T14:23:12.358 INFO:tasks.rados.rados.0.smithi026.stdout:1240: seq_num 411 ranges {3555328=491520,4046848=753664,4800512=688128,5488640=163840} 2024-09-13T14:23:12.381 INFO:tasks.rados.rados.0.smithi026.stdout:1240: writing smithi02683976-5 from 3555328 to 4046848 tid 1 2024-09-13T14:23:12.385 INFO:tasks.rados.rados.0.smithi026.stdout:1240: writing smithi02683976-5 from 4046848 to 4800512 tid 2 2024-09-13T14:23:12.389 INFO:tasks.rados.rados.0.smithi026.stdout:1240: writing smithi02683976-5 from 4800512 to 5488640 tid 3 2024-09-13T14:23:12.391 INFO:tasks.rados.rados.0.smithi026.stdout:1240: writing smithi02683976-5 from 5488640 to 5652480 tid 4 2024-09-13T14:23:12.391 INFO:tasks.rados.rados.0.smithi026.stdout:1241: copy_from oid 8 from oid 43 current snap is 116 2024-09-13T14:23:12.391 INFO:tasks.rados.rados.0.smithi026.stdout:1231: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:23:12.396 INFO:tasks.rados.rados.0.smithi026.stdout:1229: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1229: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1231: done (13 left) 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1242: rollback oid 30 current snap is 116 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 112 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1229: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 291 (ObjNum 372 snap 104 seq_num 372) dirty exists 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 1 to smithi02683976-50 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 2 to smithi02683976-50 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 3 to smithi02683976-50 2024-09-13T14:23:12.397 INFO:tasks.rados.rados.0.smithi026.stdout:1226: expect (ObjNum 307 snap 83 seq_num 307) 2024-09-13T14:23:12.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:12 smithi026 ceph-mon[99461]: pgmap v46: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 758 KiB/s wr, 7 op/s 2024-09-13T14:23:12.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:12 smithi026 ceph-mon[99461]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:23:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:12 smithi026 ceph-mon[97022]: pgmap v46: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 758 KiB/s wr, 7 op/s 2024-09-13T14:23:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:12 smithi026 ceph-mon[97022]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:23:12.700 INFO:tasks.rados.rados.0.smithi026.stdout:1235: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 4 to smithi02683976-50 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 5 to smithi02683976-50 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 6 to smithi02683976-50 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:1234: finishing write tid 7 to smithi02683976-50 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 700 (ObjNum 410 snap 116 seq_num 410) dirty exists 2024-09-13T14:23:12.703 INFO:tasks.rados.rados.0.smithi026.stdout:1234: left oid 50 (ObjNum 410 snap 116 seq_num 410) 2024-09-13T14:23:12.704 INFO:tasks.rados.rados.0.smithi026.stdout:1232: expect (ObjNum 407 snap 116 seq_num 407) 2024-09-13T14:23:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:12 smithi145 ceph-mon[89145]: pgmap v46: 121 pgs: 121 active+clean; 184 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 758 KiB/s wr, 7 op/s 2024-09-13T14:23:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:12 smithi145 ceph-mon[89145]: osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:23:12.791 INFO:tasks.rados.rados.0.smithi026.stdout:1236: expect (ObjNum 408 snap 116 seq_num 408) 2024-09-13T14:23:12.844 INFO:tasks.rados.rados.0.smithi026.stdout:1228: expect (ObjNum 336 snap 91 seq_num 336) 2024-09-13T14:23:13.029 INFO:tasks.rados.rados.0.smithi026.stdout:1227: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:23:13.029 INFO:tasks.rados.rados.0.smithi026.stdout:1227: finishing copy_from to smithi02683976-18 2024-09-13T14:23:13.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 347 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-13T14:23:13.029 INFO:tasks.rados.rados.0.smithi026.stdout:1233: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:23:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:13 smithi145 ceph-mon[89145]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:23:13.455 INFO:tasks.rados.rados.0.smithi026.stdout:1239: expect (ObjNum 356 snap 99 seq_num 356) 2024-09-13T14:23:13.487 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 1 to smithi02683976-5 2024-09-13T14:23:13.487 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 2 to smithi02683976-5 2024-09-13T14:23:13.487 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 3 to smithi02683976-5 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 4 to smithi02683976-5 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1226: done (13 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1227: done (12 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1228: done (11 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1229: done (10 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1232: done (9 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1233: done (8 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1234: done (7 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1235: done (6 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1236: done (5 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1238: done (4 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1239: done (3 left) 2024-09-13T14:23:13.488 INFO:tasks.rados.rados.0.smithi026.stdout:1243: copy_from oid 31 from oid 2 current snap is 116 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 5 to smithi02683976-5 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1240: finishing write tid 6 to smithi02683976-5 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 677 (ObjNum 411 snap 116 seq_num 411) dirty exists 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1240: left oid 5 (ObjNum 411 snap 116 seq_num 411) 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1242: finishing rollback tid 0 to smithi02683976-30 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1242: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1242: finishing rollback tid 2 to smithi02683976-30 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 304 (ObjNum 366 snap 102 seq_num 366) dirty exists 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1241: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1241: finishing copy_from to smithi02683976-8 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 574 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1240: done (3 left) 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1241: done (2 left) 2024-09-13T14:23:13.489 INFO:tasks.rados.rados.0.smithi026.stdout:1242: done (1 left) 2024-09-13T14:23:13.490 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 116 2024-09-13T14:23:13.490 INFO:tasks.rados.rados.0.smithi026.stdout:1244: seq_num 412 ranges {0=507904,507904=630784,1138688=491520,1630208=483328} 2024-09-13T14:23:13.490 INFO:tasks.rados.rados.0.smithi026.stdout:1244: writing smithi02683976-7 from 0 to 507904 tid 1 2024-09-13T14:23:13.493 INFO:tasks.rados.rados.0.smithi026.stdout:1244: writing smithi02683976-7 from 507904 to 1138688 tid 2 2024-09-13T14:23:13.495 INFO:tasks.rados.rados.0.smithi026.stdout:1244: writing smithi02683976-7 from 1138688 to 1630208 tid 3 2024-09-13T14:23:13.498 INFO:tasks.rados.rados.0.smithi026.stdout:1244: writing smithi02683976-7 from 1630208 to 2113536 tid 4 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1245: snap_create 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1243: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 1 to smithi02683976-7 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1243: finishing copy_from to smithi02683976-31 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 282 (ObjNum 328 snap 89 seq_num 328) dirty exists 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 2 to smithi02683976-7 2024-09-13T14:23:13.499 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 3 to smithi02683976-7 2024-09-13T14:23:13.500 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 4 to smithi02683976-7 2024-09-13T14:23:13.501 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 5 to smithi02683976-7 2024-09-13T14:23:13.502 INFO:tasks.rados.rados.0.smithi026.stdout:1244: finishing write tid 6 to smithi02683976-7 2024-09-13T14:23:13.502 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 502 (ObjNum 412 snap 116 seq_num 412) dirty exists 2024-09-13T14:23:13.502 INFO:tasks.rados.rados.0.smithi026.stdout:1244: left oid 7 (ObjNum 412 snap 116 seq_num 412) 2024-09-13T14:23:13.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:13 smithi026 ceph-mon[99461]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:23:13.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:13 smithi026 ceph-mon[97022]: osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1243: done (2 left) 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1244: done (1 left) 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1245: done (0 left) 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1246: copy_from oid 31 from oid 47 current snap is 117 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1247: read oid 12 snap 112 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1247: expect (ObjNum 389 snap 110 seq_num 389) 2024-09-13T14:23:13.889 INFO:tasks.rados.rados.0.smithi026.stdout:1248: delete oid 32 current snap is 117 2024-09-13T14:23:13.894 INFO:tasks.rados.rados.0.smithi026.stdout:1248: done (2 left) 2024-09-13T14:23:13.894 INFO:tasks.rados.rados.0.smithi026.stdout:1249: snap_create 2024-09-13T14:23:13.907 INFO:tasks.rados.rados.0.smithi026.stdout:1246: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:23:13.907 INFO:tasks.rados.rados.0.smithi026.stdout:1246: finishing copy_from to smithi02683976-31 2024-09-13T14:23:13.907 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 291 (ObjNum 372 snap 104 seq_num 372) dirty exists 2024-09-13T14:23:13.928 INFO:tasks.rados.rados.0.smithi026.stdout:1247: expect (ObjNum 389 snap 110 seq_num 389) 2024-09-13T14:23:14.360 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:23:14.070+0000 7f82de253640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:23:14.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: pgmap v49: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-13T14:23:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:23:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: pgmap v49: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: pgmap v49: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:14.890 INFO:tasks.rados.rados.0.smithi026.stdout:1246: done (2 left) 2024-09-13T14:23:14.891 INFO:tasks.rados.rados.0.smithi026.stdout:1247: done (1 left) 2024-09-13T14:23:14.891 INFO:tasks.rados.rados.0.smithi026.stdout:1249: done (0 left) 2024-09-13T14:23:14.891 INFO:tasks.rados.rados.0.smithi026.stdout:1250: copy_from oid 38 from oid 8 current snap is 118 2024-09-13T14:23:14.891 INFO:tasks.rados.rados.0.smithi026.stdout:1251: delete oid 42 current snap is 118 2024-09-13T14:23:14.894 INFO:tasks.rados.rados.0.smithi026.stdout:1251: done (1 left) 2024-09-13T14:23:14.894 INFO:tasks.rados.rados.0.smithi026.stdout:1252: delete oid 29 current snap is 118 2024-09-13T14:23:14.897 INFO:tasks.rados.rados.0.smithi026.stdout:1252: done (1 left) 2024-09-13T14:23:14.897 INFO:tasks.rados.rados.0.smithi026.stdout:1253: rollback oid 27 current snap is 118 2024-09-13T14:23:14.897 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 98 2024-09-13T14:23:14.898 INFO:tasks.rados.rados.0.smithi026.stdout:1254: snap_remove snap 117 2024-09-13T14:23:15.895 INFO:tasks.rados.rados.0.smithi026.stdout:1254: done (2 left) 2024-09-13T14:23:15.895 INFO:tasks.rados.rados.0.smithi026.stdout:1255: snap_remove snap 114 2024-09-13T14:23:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:15 smithi026 ceph-mon[99461]: pgmap v51: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 23 op/s 2024-09-13T14:23:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:15 smithi026 ceph-mon[99461]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:23:15.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:15 smithi026 ceph-mon[97022]: pgmap v51: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 23 op/s 2024-09-13T14:23:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:15 smithi026 ceph-mon[97022]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:23:16.044 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:15 smithi145 ceph-mon[89145]: pgmap v51: 121 pgs: 121 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 23 op/s 2024-09-13T14:23:16.045 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:15 smithi145 ceph-mon[89145]: osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:23:16.045 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:16.045 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:16.901 INFO:tasks.rados.rados.0.smithi026.stdout:1255: done (2 left) 2024-09-13T14:23:16.901 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 118 2024-09-13T14:23:16.901 INFO:tasks.rados.rados.0.smithi026.stdout:1256: seq_num 416 ranges {3571712=573440,4145152=573440,4718592=475136,5193728=557056} 2024-09-13T14:23:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:16 smithi026 ceph-mon[97022]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:23:16.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:16 smithi026 ceph-mon[99461]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:23:16.924 INFO:tasks.rados.rados.0.smithi026.stdout:1256: writing smithi02683976-44 from 3571712 to 4145152 tid 1 2024-09-13T14:23:16.927 INFO:tasks.rados.rados.0.smithi026.stdout:1256: writing smithi02683976-44 from 4145152 to 4718592 tid 2 2024-09-13T14:23:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:1256: writing smithi02683976-44 from 4718592 to 5193728 tid 3 2024-09-13T14:23:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1256: writing smithi02683976-44 from 5193728 to 5750784 tid 4 2024-09-13T14:23:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1257: rollback oid 13 current snap is 118 2024-09-13T14:23:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 109 2024-09-13T14:23:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1253: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:1253: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:1253: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 681 (ObjNum 295 snap 82 seq_num 295) dirty exists 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:1250: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:1250: finishing copy_from to smithi02683976-38 2024-09-13T14:23:16.936 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 574 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 1 to smithi02683976-44 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 2 to smithi02683976-44 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 3 to smithi02683976-44 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1250: done (3 left) 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1253: done (2 left) 2024-09-13T14:23:16.937 INFO:tasks.rados.rados.0.smithi026.stdout:1258: snap_remove snap 98 2024-09-13T14:23:17.036 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:16 smithi145 ceph-mon[89145]: osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:23:17.037 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:17.905 INFO:tasks.rados.rados.0.smithi026.stdout:1258: done (2 left) 2024-09-13T14:23:17.905 INFO:tasks.rados.rados.0.smithi026.stdout:1259: copy_from oid 4 from oid 8 current snap is 118 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1257: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 606 (ObjNum 129 snap 0 seq_num 519651984) dirty dne 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 4 to smithi02683976-44 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 5 to smithi02683976-44 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1256: finishing write tid 6 to smithi02683976-44 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 435 (ObjNum 416 snap 118 seq_num 416) dirty exists 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1256: left oid 44 (ObjNum 416 snap 118 seq_num 416) 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1256: done (2 left) 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1257: done (1 left) 2024-09-13T14:23:17.906 INFO:tasks.rados.rados.0.smithi026.stdout:1260: snap_create 2024-09-13T14:23:17.923 INFO:tasks.rados.rados.0.smithi026.stdout:1259: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:23:17.924 INFO:tasks.rados.rados.0.smithi026.stdout:1259: finishing copy_from to smithi02683976-4 2024-09-13T14:23:17.925 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 574 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: pgmap v54: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 7 op/s 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:17.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:17 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:18.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: pgmap v54: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 7 op/s 2024-09-13T14:23:18.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: pgmap v54: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 7 op/s 2024-09-13T14:23:18.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:18.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:17 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:18.910 INFO:tasks.rados.rados.0.smithi026.stdout:1259: done (1 left) 2024-09-13T14:23:18.910 INFO:tasks.rados.rados.0.smithi026.stdout:1260: done (0 left) 2024-09-13T14:23:18.910 INFO:tasks.rados.rados.0.smithi026.stdout:1261: copy_from oid 35 from oid 22 current snap is 119 2024-09-13T14:23:18.910 INFO:tasks.rados.rados.0.smithi026.stdout:1262: rollback oid 20 current snap is 119 2024-09-13T14:23:18.910 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 118 2024-09-13T14:23:18.911 INFO:tasks.rados.rados.0.smithi026.stdout:1263: delete oid 27 current snap is 119 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:1263: done (2 left) 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:1264: rollback oid 36 current snap is 119 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 111 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:1265: delete oid 7 current snap is 119 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:1262: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:23:18.917 INFO:tasks.rados.rados.0.smithi026.stdout:1262: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:23:18.918 INFO:tasks.rados.rados.0.smithi026.stdout:1262: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:23:18.918 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 658 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-13T14:23:18.919 INFO:tasks.rados.rados.0.smithi026.stdout:1262: done (3 left) 2024-09-13T14:23:18.920 INFO:tasks.rados.rados.0.smithi026.stdout:1265: done (2 left) 2024-09-13T14:23:18.920 INFO:tasks.rados.rados.0.smithi026.stdout:1266: copy_from oid 20 from oid 34 current snap is 119 2024-09-13T14:23:18.920 INFO:tasks.rados.rados.0.smithi026.stdout:1264: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:23:18.920 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 119 2024-09-13T14:23:18.920 INFO:tasks.rados.rados.0.smithi026.stdout:1267: seq_num 419 ranges {0=622592,622592=671744,1294336=647168,1941504=286720} 2024-09-13T14:23:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1267: writing smithi02683976-7 from 0 to 622592 tid 1 2024-09-13T14:23:18.927 INFO:tasks.rados.rados.0.smithi026.stdout:1267: writing smithi02683976-7 from 622592 to 1294336 tid 2 2024-09-13T14:23:18.931 INFO:tasks.rados.rados.0.smithi026.stdout:1267: writing smithi02683976-7 from 1294336 to 1941504 tid 3 2024-09-13T14:23:18.933 INFO:tasks.rados.rados.0.smithi026.stdout:1267: writing smithi02683976-7 from 1941504 to 2228224 tid 4 2024-09-13T14:23:18.934 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 119 2024-09-13T14:23:18.934 INFO:tasks.rados.rados.0.smithi026.stdout:1264: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:23:18.934 INFO:tasks.rados.rados.0.smithi026.stdout:1268: seq_num 420 ranges {0=638976,638976=557056,1196032=450560,1646592=598016} 2024-09-13T14:23:18.938 INFO:tasks.rados.rados.0.smithi026.stdout:1268: writing smithi02683976-3 from 0 to 638976 tid 1 2024-09-13T14:23:18.942 INFO:tasks.rados.rados.0.smithi026.stdout:1268: writing smithi02683976-3 from 638976 to 1196032 tid 2 2024-09-13T14:23:18.944 INFO:tasks.rados.rados.0.smithi026.stdout:1268: writing smithi02683976-3 from 1196032 to 1646592 tid 3 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:1268: writing smithi02683976-3 from 1646592 to 2244608 tid 4 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:1269: rollback oid 6 current snap is 119 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 112 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:1270: read oid 49 snap -1 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:1270: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:18.951 INFO:tasks.rados.rados.0.smithi026.stdout:1271: delete oid 8 current snap is 119 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1264: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 711 (ObjNum 388 snap 110 seq_num 388) dirty exists 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 1 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 2 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 3 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 4 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 5 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: finishing write tid 6 to smithi02683976-7 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 508 (ObjNum 419 snap 119 seq_num 419) dirty exists 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1267: left oid 7 (ObjNum 419 snap 119 seq_num 419) 2024-09-13T14:23:18.952 INFO:tasks.rados.rados.0.smithi026.stdout:1269: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:23:18.953 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 0 (ObjNum 33 snap 0 seq_num 509658488) dirty dne 2024-09-13T14:23:18.955 INFO:tasks.rados.rados.0.smithi026.stdout:1264: done (7 left) 2024-09-13T14:23:18.955 INFO:tasks.rados.rados.0.smithi026.stdout:1267: done (6 left) 2024-09-13T14:23:18.955 INFO:tasks.rados.rados.0.smithi026.stdout:1269: done (5 left) 2024-09-13T14:23:18.956 INFO:tasks.rados.rados.0.smithi026.stdout:1271: done (4 left) 2024-09-13T14:23:18.956 INFO:tasks.rados.rados.0.smithi026.stdout:1272: read oid 1 snap -1 2024-09-13T14:23:18.956 INFO:tasks.rados.rados.0.smithi026.stdout:1272: expect deleted 2024-09-13T14:23:18.956 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 119 2024-09-13T14:23:18.956 INFO:tasks.rados.rados.0.smithi026.stdout:1273: seq_num 422 ranges {0=671744,671744=729088,1400832=417792,1818624=458752} 2024-09-13T14:23:18.959 INFO:tasks.rados.rados.0.smithi026.stdout:1273: writing smithi02683976-11 from 0 to 671744 tid 1 2024-09-13T14:23:18.964 INFO:tasks.rados.rados.0.smithi026.stdout:1273: writing smithi02683976-11 from 671744 to 1400832 tid 2 2024-09-13T14:23:18.968 INFO:tasks.rados.rados.0.smithi026.stdout:1273: writing smithi02683976-11 from 1400832 to 1818624 tid 3 2024-09-13T14:23:18.971 INFO:tasks.rados.rados.0.smithi026.stdout:1273: writing smithi02683976-11 from 1818624 to 2277376 tid 4 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1274: snap_create 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1261: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1261: finishing copy_from to smithi02683976-35 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 563 (ObjNum 380 snap 106 seq_num 380) dirty exists 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 1 to smithi02683976-3 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 1 to smithi02683976-11 2024-09-13T14:23:18.972 INFO:tasks.rados.rados.0.smithi026.stdout:1270: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:19.108 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 2 to smithi02683976-3 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 3 to smithi02683976-3 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1266: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 2 to smithi02683976-11 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 4 to smithi02683976-3 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1266: finishing copy_from to smithi02683976-20 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 377 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 3 to smithi02683976-11 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 5 to smithi02683976-3 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1268: finishing write tid 6 to smithi02683976-3 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 718 (ObjNum 420 snap 119 seq_num 420) dirty exists 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1268: left oid 3 (ObjNum 420 snap 119 seq_num 420) 2024-09-13T14:23:19.109 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 4 to smithi02683976-11 2024-09-13T14:23:19.110 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 5 to smithi02683976-11 2024-09-13T14:23:19.110 INFO:tasks.rados.rados.0.smithi026.stdout:1273: finishing write tid 6 to smithi02683976-11 2024-09-13T14:23:19.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 723 (ObjNum 422 snap 119 seq_num 422) dirty exists 2024-09-13T14:23:19.110 INFO:tasks.rados.rados.0.smithi026.stdout:1273: left oid 11 (ObjNum 422 snap 119 seq_num 422) 2024-09-13T14:23:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[99461]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:23:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[99461]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:23:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[99461]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:23:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[97022]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:23:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[97022]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:23:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:18 smithi026 ceph-mon[97022]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:23:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:18 smithi145 ceph-mon[89145]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:23:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:18 smithi145 ceph-mon[89145]: Reconfiguring daemon mon.a on smithi026 2024-09-13T14:23:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:18 smithi145 ceph-mon[89145]: osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1261: done (6 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1266: done (5 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1268: done (4 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1270: done (3 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1272: done (2 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1273: done (1 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1274: done (0 left) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1275: read oid 49 snap -1 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:1275: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:19.913 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 120 2024-09-13T14:23:19.914 INFO:tasks.rados.rados.0.smithi026.stdout:1276: seq_num 423 ranges {704512=688128,1392640=614400,2007040=622592,2629632=368640} 2024-09-13T14:23:19.920 INFO:tasks.rados.rados.0.smithi026.stdout:1276: writing smithi02683976-2 from 704512 to 1392640 tid 1 2024-09-13T14:23:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1276: writing smithi02683976-2 from 1392640 to 2007040 tid 2 2024-09-13T14:23:19.928 INFO:tasks.rados.rados.0.smithi026.stdout:1276: writing smithi02683976-2 from 2007040 to 2629632 tid 3 2024-09-13T14:23:19.930 INFO:tasks.rados.rados.0.smithi026.stdout:1276: writing smithi02683976-2 from 2629632 to 2998272 tid 4 2024-09-13T14:23:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1277: snap_create 2024-09-13T14:23:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 1 to smithi02683976-2 2024-09-13T14:23:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 2 to smithi02683976-2 2024-09-13T14:23:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 3 to smithi02683976-2 2024-09-13T14:23:19.932 INFO:tasks.rados.rados.0.smithi026.stdout:1275: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:20.069 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 4 to smithi02683976-2 2024-09-13T14:23:20.069 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 5 to smithi02683976-2 2024-09-13T14:23:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:1276: finishing write tid 6 to smithi02683976-2 2024-09-13T14:23:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 429 (ObjNum 423 snap 120 seq_num 423) dirty exists 2024-09-13T14:23:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:1276: left oid 2 (ObjNum 423 snap 120 seq_num 423) 2024-09-13T14:23:20.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: pgmap v57: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T14:23:20.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:23:20.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[97022]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: pgmap v57: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:23:20.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:20.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:20.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:20.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:19 smithi026 ceph-mon[99461]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: pgmap v57: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 3.1 MiB/s wr, 8 op/s 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:23:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:20.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:20.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:20.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:19 smithi145 ceph-mon[89145]: Reconfiguring daemon mgr.y on smithi026 2024-09-13T14:23:20.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:20.918 INFO:tasks.rados.rados.0.smithi026.stdout:1275: done (2 left) 2024-09-13T14:23:20.918 INFO:tasks.rados.rados.0.smithi026.stdout:1276: done (1 left) 2024-09-13T14:23:20.919 INFO:tasks.rados.rados.0.smithi026.stdout:1277: done (0 left) 2024-09-13T14:23:20.919 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 121 2024-09-13T14:23:20.919 INFO:tasks.rados.rados.0.smithi026.stdout:1278: seq_num 424 ranges {0=704512,704512=491520,1196032=434176,1630208=679936} 2024-09-13T14:23:20.922 INFO:tasks.rados.rados.0.smithi026.stdout:1278: writing smithi02683976-42 from 0 to 704512 tid 1 2024-09-13T14:23:20.926 INFO:tasks.rados.rados.0.smithi026.stdout:1278: writing smithi02683976-42 from 704512 to 1196032 tid 2 2024-09-13T14:23:20.931 INFO:tasks.rados.rados.0.smithi026.stdout:1278: writing smithi02683976-42 from 1196032 to 1630208 tid 3 2024-09-13T14:23:20.935 INFO:tasks.rados.rados.0.smithi026.stdout:1278: writing smithi02683976-42 from 1630208 to 2310144 tid 4 2024-09-13T14:23:20.936 INFO:tasks.rados.rados.0.smithi026.stdout:1279: read oid 6 snap -1 2024-09-13T14:23:20.936 INFO:tasks.rados.rados.0.smithi026.stdout:1279: expect deleted 2024-09-13T14:23:20.936 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 121 2024-09-13T14:23:20.936 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 1 to smithi02683976-42 2024-09-13T14:23:20.936 INFO:tasks.rados.rados.0.smithi026.stdout:1280: seq_num 425 ranges {4882432=720896,5603328=778240,6381568=638976,7020544=188416} 2024-09-13T14:23:20.966 INFO:tasks.rados.rados.0.smithi026.stdout:1280: writing smithi02683976-14 from 4882432 to 5603328 tid 1 2024-09-13T14:23:20.972 INFO:tasks.rados.rados.0.smithi026.stdout:1280: writing smithi02683976-14 from 5603328 to 6381568 tid 2 2024-09-13T14:23:20.975 INFO:tasks.rados.rados.0.smithi026.stdout:1280: writing smithi02683976-14 from 6381568 to 7020544 tid 3 2024-09-13T14:23:20.977 INFO:tasks.rados.rados.0.smithi026.stdout:1280: writing smithi02683976-14 from 7020544 to 7208960 tid 4 2024-09-13T14:23:20.977 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 121 2024-09-13T14:23:20.977 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 2 to smithi02683976-42 2024-09-13T14:23:20.977 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 3 to smithi02683976-42 2024-09-13T14:23:20.977 INFO:tasks.rados.rados.0.smithi026.stdout:1281: seq_num 426 ranges {8880128=737280,9617408=712704,10330112=802816,11132928=90112} 2024-09-13T14:23:21.029 INFO:tasks.rados.rados.0.smithi026.stdout:1281: writing smithi02683976-12 from 8880128 to 9617408 tid 1 2024-09-13T14:23:21.033 INFO:tasks.rados.rados.0.smithi026.stdout:1281: writing smithi02683976-12 from 9617408 to 10330112 tid 2 2024-09-13T14:23:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:1281: writing smithi02683976-12 from 10330112 to 11132928 tid 3 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1281: writing smithi02683976-12 from 11132928 to 11223040 tid 4 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1282: rmattr oid 16 current snap is 121 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1282: done (4 left) 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1283: delete oid 29 current snap is 121 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 4 to smithi02683976-42 2024-09-13T14:23:21.039 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 5 to smithi02683976-42 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1278: finishing write tid 6 to smithi02683976-42 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 306 (ObjNum 424 snap 121 seq_num 424) dirty exists 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1278: left oid 42 (ObjNum 424 snap 121 seq_num 424) 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 1 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 2 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 3 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 4 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 5 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: finishing write tid 6 to smithi02683976-14 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 579 (ObjNum 425 snap 121 seq_num 425) dirty exists 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1280: left oid 14 (ObjNum 425 snap 121 seq_num 425) 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 1 to smithi02683976-12 2024-09-13T14:23:21.040 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 2 to smithi02683976-12 2024-09-13T14:23:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:1278: done (4 left) 2024-09-13T14:23:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:1279: done (3 left) 2024-09-13T14:23:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:1280: done (2 left) 2024-09-13T14:23:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:1283: done (1 left) 2024-09-13T14:23:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:1284: snap_create 2024-09-13T14:23:21.042 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 3 to smithi02683976-12 2024-09-13T14:23:21.042 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 4 to smithi02683976-12 2024-09-13T14:23:21.042 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 5 to smithi02683976-12 2024-09-13T14:23:21.043 INFO:tasks.rados.rados.0.smithi026.stdout:1281: finishing write tid 6 to smithi02683976-12 2024-09-13T14:23:21.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 588 (ObjNum 426 snap 121 seq_num 426) dirty exists 2024-09-13T14:23:21.044 INFO:tasks.rados.rados.0.smithi026.stdout:1281: left oid 12 (ObjNum 426 snap 121 seq_num 426) 2024-09-13T14:23:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:23:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:21.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:20 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:20 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:21.574 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:21] "GET /metrics HTTP/1.1" 200 36058 "" "Prometheus/2.51.0" 2024-09-13T14:23:21.923 INFO:tasks.rados.rados.0.smithi026.stdout:1281: done (1 left) 2024-09-13T14:23:21.923 INFO:tasks.rados.rados.0.smithi026.stdout:1284: done (0 left) 2024-09-13T14:23:21.923 INFO:tasks.rados.rados.0.smithi026.stdout:1285: read oid 10 snap -1 2024-09-13T14:23:21.923 INFO:tasks.rados.rados.0.smithi026.stdout:1285: expect (ObjNum 399 snap 114 seq_num 399) 2024-09-13T14:23:21.923 INFO:tasks.rados.rados.0.smithi026.stdout:1286: delete oid 19 current snap is 122 2024-09-13T14:23:21.928 INFO:tasks.rados.rados.0.smithi026.stdout:1286: done (1 left) 2024-09-13T14:23:21.928 INFO:tasks.rados.rados.0.smithi026.stdout:1287: delete oid 5 current snap is 122 2024-09-13T14:23:21.931 INFO:tasks.rados.rados.0.smithi026.stdout:1287: done (1 left) 2024-09-13T14:23:21.931 INFO:tasks.rados.rados.0.smithi026.stdout:1288: read oid 9 snap -1 2024-09-13T14:23:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:1288: expect deleted 2024-09-13T14:23:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 122 2024-09-13T14:23:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:1289: seq_num 430 ranges {3416064=8192} 2024-09-13T14:23:21.951 INFO:tasks.rados.rados.0.smithi026.stdout:1289: writing smithi02683976-50 from 3416064 to 3424256 tid 1 2024-09-13T14:23:21.951 INFO:tasks.rados.rados.0.smithi026.stdout:1290: rmattr oid 45 current snap is 122 2024-09-13T14:23:21.951 INFO:tasks.rados.rados.0.smithi026.stdout:1290: done (3 left) 2024-09-13T14:23:21.951 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 122 2024-09-13T14:23:21.951 INFO:tasks.rados.rados.0.smithi026.stdout:1291: seq_num 431 ranges {4808704=24576} 2024-09-13T14:23:21.978 INFO:tasks.rados.rados.0.smithi026.stdout:1291: writing smithi02683976-23 from 4808704 to 4833280 tid 1 2024-09-13T14:23:21.978 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 122 2024-09-13T14:23:21.978 INFO:tasks.rados.rados.0.smithi026.stdout:1292: seq_num 432 ranges {0=40960} 2024-09-13T14:23:21.978 INFO:tasks.rados.rados.0.smithi026.stdout:1292: writing smithi02683976-5 from 0 to 40960 tid 1 2024-09-13T14:23:21.978 INFO:tasks.rados.rados.0.smithi026.stdout:1293: snap_remove snap 119 2024-09-13T14:23:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:21 smithi145 ceph-mon[89145]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:21 smithi145 ceph-mon[89145]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:23:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:21 smithi145 ceph-mon[89145]: pgmap v60: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 858 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:23:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:21 smithi145 ceph-mon[89145]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:23:22.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[99461]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:22.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[99461]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:23:22.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[99461]: pgmap v60: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 858 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:23:22.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[99461]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:23:22.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[97022]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:23:22.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[97022]: Reconfiguring daemon mon.c on smithi026 2024-09-13T14:23:22.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[97022]: pgmap v60: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 858 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:23:22.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:21 smithi026 ceph-mon[97022]: osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:23:22.930 INFO:tasks.rados.rados.0.smithi026.stdout:1293: done (5 left) 2024-09-13T14:23:22.930 INFO:tasks.rados.rados.0.smithi026.stdout:1294: snap_remove snap 121 2024-09-13T14:23:23.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:23:23.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.0 on smithi026 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:23.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:22 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.0 on smithi026 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:22 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.0 on smithi026 2024-09-13T14:23:23.892 INFO:tasks.rados.rados.0.smithi026.stdout:1294: done (5 left) 2024-09-13T14:23:23.892 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 122 2024-09-13T14:23:23.892 INFO:tasks.rados.rados.0.smithi026.stdout:1295: seq_num 433 ranges {4038656=57344} 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1295: writing smithi02683976-18 from 4038656 to 4096000 tid 1 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1296: read oid 45 snap 112 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1296: expect deleted 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1297: rmattr oid 46 current snap is 122 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1297: done (7 left) 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 122 2024-09-13T14:23:23.916 INFO:tasks.rados.rados.0.smithi026.stdout:1285: expect (ObjNum 399 snap 114 seq_num 399) 2024-09-13T14:23:24.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[99461]: pgmap v63: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:24.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[99461]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:23:24.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[99461]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:23:24.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[97022]: pgmap v63: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:24.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[97022]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:23:24.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:23 smithi026 ceph-mon[97022]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:23:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:23 smithi145 ceph-mon[89145]: pgmap v63: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:23 smithi145 ceph-mon[89145]: osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:23:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:23 smithi145 ceph-mon[89145]: osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:23:24.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:24.273 INFO:tasks.rados.rados.0.smithi026.stdout:1289: finishing write tid 1 to smithi02683976-50 2024-09-13T14:23:24.273 INFO:tasks.rados.rados.0.smithi026.stdout:1289: finishing write tid 2 to smithi02683976-50 2024-09-13T14:23:24.273 INFO:tasks.rados.rados.0.smithi026.stdout:1289: finishing write tid 3 to smithi02683976-50 2024-09-13T14:23:24.273 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 726 (ObjNum 430 snap 122 seq_num 430) dirty exists 2024-09-13T14:23:24.273 INFO:tasks.rados.rados.0.smithi026.stdout:1289: left oid 50 (ObjNum 430 snap 122 seq_num 430) 2024-09-13T14:23:24.274 INFO:tasks.rados.rados.0.smithi026.stdout:1292: finishing write tid 1 to smithi02683976-5 2024-09-13T14:23:24.274 INFO:tasks.rados.rados.0.smithi026.stdout:1292: finishing write tid 2 to smithi02683976-5 2024-09-13T14:23:24.274 INFO:tasks.rados.rados.0.smithi026.stdout:1291: finishing write tid 1 to smithi02683976-23 2024-09-13T14:23:24.274 INFO:tasks.rados.rados.0.smithi026.stdout:1291: finishing write tid 2 to smithi02683976-23 2024-09-13T14:23:24.274 INFO:tasks.rados.rados.0.smithi026.stdout:1298: seq_num 434 ranges {8413184=73728} 2024-09-13T14:23:24.320 INFO:tasks.rados.rados.0.smithi026.stdout:1298: writing smithi02683976-34 from 8413184 to 8486912 tid 1 2024-09-13T14:23:24.320 INFO:tasks.rados.rados.0.smithi026.stdout:1285: done (7 left) 2024-09-13T14:23:24.320 INFO:tasks.rados.rados.0.smithi026.stdout:1288: done (6 left) 2024-09-13T14:23:24.320 INFO:tasks.rados.rados.0.smithi026.stdout:1289: done (5 left) 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1299: read oid 26 snap -1 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1299: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1300: read oid 38 snap -1 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1300: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1301: rollback oid 14 current snap is 122 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 116 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1292: finishing write tid 3 to smithi02683976-5 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 722 (ObjNum 432 snap 122 seq_num 432) dirty exists 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1292: left oid 5 (ObjNum 432 snap 122 seq_num 432) 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1291: finishing write tid 3 to smithi02683976-23 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 582 (ObjNum 431 snap 122 seq_num 431) dirty exists 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1291: left oid 23 (ObjNum 431 snap 122 seq_num 431) 2024-09-13T14:23:24.321 INFO:tasks.rados.rados.0.smithi026.stdout:1295: finishing write tid 1 to smithi02683976-18 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1291: done (7 left) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1292: done (6 left) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1296: done (5 left) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1302: rmattr oid 12 current snap is 122 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1302: done (5 left) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1303: setattr oid 20 current snap is 122 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1295: finishing write tid 2 to smithi02683976-18 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1295: finishing write tid 3 to smithi02683976-18 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 589 (ObjNum 433 snap 122 seq_num 433) dirty exists 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1295: left oid 18 (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1295: done (5 left) 2024-09-13T14:23:24.322 INFO:tasks.rados.rados.0.smithi026.stdout:1304: snap_remove snap 118 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:1298: finishing write tid 1 to smithi02683976-34 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:1298: finishing write tid 2 to smithi02683976-34 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 663 (ObjNum 234 snap 58 seq_num 234) dirty exists 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:1298: finishing write tid 3 to smithi02683976-34 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 397 (ObjNum 434 snap 122 seq_num 434) dirty exists 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:1298: left oid 34 (ObjNum 434 snap 122 seq_num 434) 2024-09-13T14:23:24.964 INFO:tasks.rados.rados.0.smithi026.stdout:1300: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:23:25.114 INFO:tasks.rados.rados.0.smithi026.stdout:1301: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1301: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1301: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 594 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1298: done (5 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1300: done (4 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1301: done (3 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1303: done (2 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1304: done (1 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1305: rmattr oid 44 current snap is 122 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1305: done (1 left) 2024-09-13T14:23:25.115 INFO:tasks.rados.rados.0.smithi026.stdout:1306: snap_remove snap 109 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:24 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.1 on smithi026 2024-09-13T14:23:25.393 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.1 on smithi026 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:25.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:23:25.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:23:25.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:25.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:24 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.1 on smithi026 2024-09-13T14:23:25.964 INFO:tasks.rados.rados.0.smithi026.stdout:1306: done (1 left) 2024-09-13T14:23:25.964 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 122 2024-09-13T14:23:25.964 INFO:tasks.rados.rados.0.smithi026.stdout:1299: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: pgmap v66: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:26.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: pgmap v66: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:26.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: pgmap v66: 121 pgs: 121 active+clean; 209 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.0 MiB/s wr, 17 op/s 2024-09-13T14:23:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:23:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:26.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:26.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:26.399 INFO:tasks.rados.rados.0.smithi026.stdout:1307: seq_num 435 ranges {1843200=90112} 2024-09-13T14:23:26.410 INFO:tasks.rados.rados.0.smithi026.stdout:1307: writing smithi02683976-31 from 1843200 to 1933312 tid 1 2024-09-13T14:23:26.410 INFO:tasks.rados.rados.0.smithi026.stdout:1299: done (1 left) 2024-09-13T14:23:26.410 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 122 2024-09-13T14:23:26.410 INFO:tasks.rados.rados.0.smithi026.stdout:1308: seq_num 436 ranges {0=106496} 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:1308: writing smithi02683976-29 from 0 to 106496 tid 1 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:1309: rollback oid 12 current snap is 122 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 112 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:1310: rollback oid 8 current snap is 122 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 111 2024-09-13T14:23:26.411 INFO:tasks.rados.rados.0.smithi026.stdout:1311: rmattr oid 35 current snap is 122 2024-09-13T14:23:26.412 INFO:tasks.rados.rados.0.smithi026.stdout:1312: rmattr oid 13 current snap is 122 2024-09-13T14:23:26.412 INFO:tasks.rados.rados.0.smithi026.stdout:1312: done (5 left) 2024-09-13T14:23:26.412 INFO:tasks.rados.rados.0.smithi026.stdout:1313: delete oid 6 current snap is 122 2024-09-13T14:23:26.413 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 0 (ObjNum 380 snap 106 seq_num 380) dirty exists 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1309: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1310: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1308: finishing write tid 1 to smithi02683976-29 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1309: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1307: finishing write tid 1 to smithi02683976-31 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:1310: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:23:26.415 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 577 (ObjNum 318 snap 86 seq_num 318) dirty exists 2024-09-13T14:23:26.416 INFO:tasks.rados.rados.0.smithi026.stdout:1307: finishing write tid 2 to smithi02683976-31 2024-09-13T14:23:26.416 INFO:tasks.rados.rados.0.smithi026.stdout:1309: finishing rollback tid 2 to smithi02683976-12 2024-09-13T14:23:26.416 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 591 (ObjNum 389 snap 110 seq_num 389) dirty exists 2024-09-13T14:23:26.416 INFO:tasks.rados.rados.0.smithi026.stdout:1308: finishing write tid 2 to smithi02683976-29 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:1309: done (5 left) 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:1310: done (4 left) 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:1311: done (3 left) 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:1313: done (2 left) 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 122 2024-09-13T14:23:26.417 INFO:tasks.rados.rados.0.smithi026.stdout:1314: seq_num 438 ranges {1843200=139264} 2024-09-13T14:23:26.428 INFO:tasks.rados.rados.0.smithi026.stdout:1314: writing smithi02683976-47 from 1843200 to 1982464 tid 1 2024-09-13T14:23:26.429 INFO:tasks.rados.rados.0.smithi026.stdout:1315: snap_remove snap 116 2024-09-13T14:23:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1315: done (3 left) 2024-09-13T14:23:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1316: read oid 3 snap -1 2024-09-13T14:23:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1316: expect (ObjNum 420 snap 119 seq_num 420) 2024-09-13T14:23:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1317: copy_from oid 24 from oid 43 current snap is 122 2024-09-13T14:23:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1308: finishing write tid 3 to smithi02683976-29 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 673 (ObjNum 436 snap 122 seq_num 436) dirty exists 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1308: left oid 29 (ObjNum 436 snap 122 seq_num 436) 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1307: finishing write tid 3 to smithi02683976-31 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 739 (ObjNum 435 snap 122 seq_num 435) dirty exists 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1307: left oid 31 (ObjNum 435 snap 122 seq_num 435) 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1314: finishing write tid 1 to smithi02683976-47 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1314: finishing write tid 2 to smithi02683976-47 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1307: done (4 left) 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1308: done (3 left) 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1318: delete oid 32 current snap is 122 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1314: finishing write tid 3 to smithi02683976-47 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 293 (ObjNum 438 snap 122 seq_num 438) dirty exists 2024-09-13T14:23:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1314: left oid 47 (ObjNum 438 snap 122 seq_num 438) 2024-09-13T14:23:26.989 INFO:tasks.rados.rados.0.smithi026.stdout:1314: done (3 left) 2024-09-13T14:23:26.989 INFO:tasks.rados.rados.0.smithi026.stdout:1318: done (2 left) 2024-09-13T14:23:26.989 INFO:tasks.rados.rados.0.smithi026.stdout:1319: copy_from oid 23 from oid 26 current snap is 122 2024-09-13T14:23:26.989 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 122 2024-09-13T14:23:26.989 INFO:tasks.rados.rados.0.smithi026.stdout:1320: seq_num 440 ranges {4038656=172032} 2024-09-13T14:23:27.015 INFO:tasks.rados.rados.0.smithi026.stdout:1320: writing smithi02683976-40 from 4038656 to 4210688 tid 1 2024-09-13T14:23:27.015 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 122 2024-09-13T14:23:27.016 INFO:tasks.rados.rados.0.smithi026.stdout:1321: seq_num 441 ranges {6602752=188416} 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1321: writing smithi02683976-37 from 6602752 to 6791168 tid 1 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1322: read oid 18 snap -1 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1322: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1323: rollback oid 27 current snap is 122 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 112 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1317: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1324: copy_from oid 36 from oid 19 current snap is 122 2024-09-13T14:23:27.055 INFO:tasks.rados.rados.0.smithi026.stdout:1316: expect (ObjNum 420 snap 119 seq_num 420) 2024-09-13T14:23:27.120 INFO:tasks.rados.rados.0.smithi026.stdout:1317: finishing copy_from to smithi02683976-24 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 574 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:1319: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:1320: finishing write tid 1 to smithi02683976-40 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:1320: finishing write tid 2 to smithi02683976-40 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:1319: finishing copy_from to smithi02683976-23 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 575 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:1320: finishing write tid 3 to smithi02683976-40 2024-09-13T14:23:27.121 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 411 (ObjNum 440 snap 122 seq_num 440) dirty exists 2024-09-13T14:23:27.122 INFO:tasks.rados.rados.0.smithi026.stdout:1320: left oid 40 (ObjNum 440 snap 122 seq_num 440) 2024-09-13T14:23:27.122 INFO:tasks.rados.rados.0.smithi026.stdout:1323: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:23:27.122 INFO:tasks.rados.rados.0.smithi026.stdout:1324: finishing copy_from to smithi02683976-36 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1324: got expected ENOENT (src dne) 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1324: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1323: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 756 (ObjNum 393 snap 112 seq_num 393) dirty exists 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1321: finishing write tid 1 to smithi02683976-37 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1321: finishing write tid 2 to smithi02683976-37 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1321: finishing write tid 3 to smithi02683976-37 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 308 (ObjNum 441 snap 122 seq_num 441) dirty exists 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1321: left oid 37 (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:23:27.123 INFO:tasks.rados.rados.0.smithi026.stdout:1322: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:26 smithi145 ceph-mon[89145]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:23:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:26 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.2 on smithi026 2024-09-13T14:23:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:26 smithi145 ceph-mon[89145]: osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:23:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1316: done (7 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1317: done (6 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1319: done (5 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1320: done (4 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1321: done (3 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1322: done (2 left) 2024-09-13T14:23:27.389 INFO:tasks.rados.rados.0.smithi026.stdout:1323: done (1 left) 2024-09-13T14:23:27.390 INFO:tasks.rados.rados.0.smithi026.stdout:1324: done (0 left) 2024-09-13T14:23:27.390 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 122 2024-09-13T14:23:27.390 INFO:tasks.rados.rados.0.smithi026.stdout:1325: seq_num 442 ranges {3170304=204800} 2024-09-13T14:23:27.408 INFO:tasks.rados.rados.0.smithi026.stdout:1325: writing smithi02683976-38 from 3170304 to 3375104 tid 1 2024-09-13T14:23:27.408 INFO:tasks.rados.rados.0.smithi026.stdout:1326: read oid 40 snap 120 2024-09-13T14:23:27.408 INFO:tasks.rados.rados.0.smithi026.stdout:1326: expect (ObjNum 190 snap 44 seq_num 190) 2024-09-13T14:23:27.408 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 122 2024-09-13T14:23:27.408 INFO:tasks.rados.rados.0.smithi026.stdout:1327: seq_num 443 ranges {0=221184} 2024-09-13T14:23:27.409 INFO:tasks.rados.rados.0.smithi026.stdout:1327: writing smithi02683976-6 from 0 to 221184 tid 1 2024-09-13T14:23:27.409 INFO:tasks.rados.rados.0.smithi026.stdout:1328: read oid 49 snap -1 2024-09-13T14:23:27.409 INFO:tasks.rados.rados.0.smithi026.stdout:1328: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:27.409 INFO:tasks.rados.rados.0.smithi026.stdout:1329: delete oid 41 current snap is 122 2024-09-13T14:23:27.410 INFO:tasks.rados.rados.0.smithi026.stdout:1325: finishing write tid 1 to smithi02683976-38 2024-09-13T14:23:27.410 INFO:tasks.rados.rados.0.smithi026.stdout:1325: finishing write tid 2 to smithi02683976-38 2024-09-13T14:23:27.412 INFO:tasks.rados.rados.0.smithi026.stdout:1325: finishing write tid 3 to smithi02683976-38 2024-09-13T14:23:27.412 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 625 (ObjNum 442 snap 122 seq_num 442) dirty exists 2024-09-13T14:23:27.412 INFO:tasks.rados.rados.0.smithi026.stdout:1325: left oid 38 (ObjNum 442 snap 122 seq_num 442) 2024-09-13T14:23:27.413 INFO:tasks.rados.rados.0.smithi026.stdout:1327: finishing write tid 1 to smithi02683976-6 2024-09-13T14:23:27.413 INFO:tasks.rados.rados.0.smithi026.stdout:1325: done (4 left) 2024-09-13T14:23:27.413 INFO:tasks.rados.rados.0.smithi026.stdout:1329: done (3 left) 2024-09-13T14:23:27.414 INFO:tasks.rados.rados.0.smithi026.stdout:1330: read oid 33 snap 120 2024-09-13T14:23:27.414 INFO:tasks.rados.rados.0.smithi026.stdout:1330: expect (ObjNum 519558080 snap 21981 seq_num 518903152) 2024-09-13T14:23:27.414 INFO:tasks.rados.rados.0.smithi026.stdout:1331: setattr oid 36 current snap is 122 2024-09-13T14:23:27.414 INFO:tasks.rados.rados.0.smithi026.stdout:1327: finishing write tid 2 to smithi02683976-6 2024-09-13T14:23:27.414 INFO:tasks.rados.rados.0.smithi026.stdout:1330: expect (ObjNum 519558080 snap 21981 seq_num 518903152) 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[97022]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.2 on smithi026 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[97022]: osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[99461]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:23:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.2 on smithi026 2024-09-13T14:23:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[99461]: osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:23:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1327: finishing write tid 3 to smithi02683976-6 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 746 (ObjNum 443 snap 122 seq_num 443) dirty exists 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1327: left oid 6 (ObjNum 443 snap 122 seq_num 443) 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1327: done (4 left) 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1330: done (3 left) 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1332: read oid 9 snap -1 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1332: expect deleted 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 122 2024-09-13T14:23:27.419 INFO:tasks.rados.rados.0.smithi026.stdout:1333: seq_num 445 ranges {2048000=253952} 2024-09-13T14:23:27.432 INFO:tasks.rados.rados.0.smithi026.stdout:1333: writing smithi02683976-39 from 2048000 to 2301952 tid 1 2024-09-13T14:23:27.432 INFO:tasks.rados.rados.0.smithi026.stdout:1334: read oid 18 snap -1 2024-09-13T14:23:27.432 INFO:tasks.rados.rados.0.smithi026.stdout:1334: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:27.432 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 122 2024-09-13T14:23:27.433 INFO:tasks.rados.rados.0.smithi026.stdout:1335: seq_num 446 ranges {0=278528} 2024-09-13T14:23:27.434 INFO:tasks.rados.rados.0.smithi026.stdout:1335: writing smithi02683976-32 from 0 to 278528 tid 1 2024-09-13T14:23:27.434 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 758 (ObjNum 388 snap 110 seq_num 388) dirty exists 2024-09-13T14:23:27.434 INFO:tasks.rados.rados.0.smithi026.stdout:1326: expect (ObjNum 190 snap 44 seq_num 190) 2024-09-13T14:23:27.632 INFO:tasks.rados.rados.0.smithi026.stdout:1328: expect (ObjNum 273 snap 70 seq_num 273) 2024-09-13T14:23:27.770 INFO:tasks.rados.rados.0.smithi026.stdout:1333: finishing write tid 1 to smithi02683976-39 2024-09-13T14:23:27.771 INFO:tasks.rados.rados.0.smithi026.stdout:1333: finishing write tid 2 to smithi02683976-39 2024-09-13T14:23:27.771 INFO:tasks.rados.rados.0.smithi026.stdout:1333: finishing write tid 3 to smithi02683976-39 2024-09-13T14:23:27.771 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 576 (ObjNum 445 snap 122 seq_num 445) dirty exists 2024-09-13T14:23:27.771 INFO:tasks.rados.rados.0.smithi026.stdout:1333: left oid 39 (ObjNum 445 snap 122 seq_num 445) 2024-09-13T14:23:27.771 INFO:tasks.rados.rados.0.smithi026.stdout:1334: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1335: finishing write tid 1 to smithi02683976-32 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1335: finishing write tid 2 to smithi02683976-32 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1326: done (6 left) 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1328: done (5 left) 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1331: done (4 left) 2024-09-13T14:23:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:1332: done (3 left) 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:1333: done (2 left) 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:1334: done (1 left) 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:1336: delete oid 13 current snap is 122 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:1335: finishing write tid 3 to smithi02683976-32 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 627 (ObjNum 446 snap 122 seq_num 446) dirty exists 2024-09-13T14:23:28.034 INFO:tasks.rados.rados.0.smithi026.stdout:1335: left oid 32 (ObjNum 446 snap 122 seq_num 446) 2024-09-13T14:23:28.035 INFO:tasks.rados.rados.0.smithi026.stdout:1335: done (1 left) 2024-09-13T14:23:28.035 INFO:tasks.rados.rados.0.smithi026.stdout:1336: done (0 left) 2024-09-13T14:23:28.035 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 122 2024-09-13T14:23:28.035 INFO:tasks.rados.rados.0.smithi026.stdout:1337: seq_num 448 ranges {4038656=311296} 2024-09-13T14:23:28.059 INFO:tasks.rados.rados.0.smithi026.stdout:1337: writing smithi02683976-49 from 4038656 to 4349952 tid 1 2024-09-13T14:23:28.059 INFO:tasks.rados.rados.0.smithi026.stdout:1338: snap_remove snap 112 2024-09-13T14:23:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: pgmap v69: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 132 KiB/s wr, 15 op/s 2024-09-13T14:23:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:23:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.3 on smithi026 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: pgmap v69: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 132 KiB/s wr, 15 op/s 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:27 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.3 on smithi026 2024-09-13T14:23:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: pgmap v69: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 132 KiB/s wr, 15 op/s 2024-09-13T14:23:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:23:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:23:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:27 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.3 on smithi026 2024-09-13T14:23:28.893 INFO:tasks.rados.rados.0.smithi026.stdout:1338: done (1 left) 2024-09-13T14:23:28.893 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 122 2024-09-13T14:23:28.893 INFO:tasks.rados.rados.0.smithi026.stdout:1337: finishing write tid 1 to smithi02683976-49 2024-09-13T14:23:28.893 INFO:tasks.rados.rados.0.smithi026.stdout:1339: seq_num 449 ranges {2301952=327680} 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:1339: writing smithi02683976-39 from 2301952 to 2629632 tid 1 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:1340: setattr oid 30 current snap is 122 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:1337: finishing write tid 2 to smithi02683976-49 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:1337: finishing write tid 3 to smithi02683976-49 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 414 (ObjNum 448 snap 122 seq_num 448) dirty exists 2024-09-13T14:23:28.910 INFO:tasks.rados.rados.0.smithi026.stdout:1337: left oid 49 (ObjNum 448 snap 122 seq_num 448) 2024-09-13T14:23:28.911 INFO:tasks.rados.rados.0.smithi026.stdout:1337: done (2 left) 2024-09-13T14:23:28.911 INFO:tasks.rados.rados.0.smithi026.stdout:1341: copy_from oid 47 from oid 9 current snap is 122 2024-09-13T14:23:28.911 INFO:tasks.rados.rados.0.smithi026.stdout:1342: snap_create 2024-09-13T14:23:28.912 INFO:tasks.rados.rados.0.smithi026.stdout:1339: finishing write tid 1 to smithi02683976-39 2024-09-13T14:23:28.912 INFO:tasks.rados.rados.0.smithi026.stdout:1339: finishing write tid 2 to smithi02683976-39 2024-09-13T14:23:28.914 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 575 (ObjNum 366 snap 102 seq_num 366) dirty exists 2024-09-13T14:23:28.914 INFO:tasks.rados.rados.0.smithi026.stdout:1339: finishing write tid 3 to smithi02683976-39 2024-09-13T14:23:28.914 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 578 (ObjNum 449 snap 122 seq_num 449) dirty exists 2024-09-13T14:23:28.914 INFO:tasks.rados.rados.0.smithi026.stdout:1339: left oid 39 (ObjNum 449 snap 122 seq_num 449) 2024-09-13T14:23:28.914 INFO:tasks.rados.rados.0.smithi026.stdout:1341: finishing copy_from to smithi02683976-47 2024-09-13T14:23:28.915 INFO:tasks.rados.rados.0.smithi026.stdout:1341: got expected ENOENT (src dne) 2024-09-13T14:23:28.915 INFO:tasks.rados.rados.0.smithi026.stdout:1341: finishing copy_from racing read to smithi02683976-47 2024-09-13T14:23:29.894 INFO:tasks.rados.rados.0.smithi026.stdout:1339: done (3 left) 2024-09-13T14:23:29.894 INFO:tasks.rados.rados.0.smithi026.stdout:1340: done (2 left) 2024-09-13T14:23:29.894 INFO:tasks.rados.rados.0.smithi026.stdout:1341: done (1 left) 2024-09-13T14:23:29.895 INFO:tasks.rados.rados.0.smithi026.stdout:1342: done (0 left) 2024-09-13T14:23:29.895 INFO:tasks.rados.rados.0.smithi026.stdout:1343: snap_remove snap 122 2024-09-13T14:23:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: Reconfiguring mds.a (monmap changed)... 2024-09-13T14:23:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: Reconfiguring daemon mds.a on smithi026 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: pgmap v71: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 110 KiB/s wr, 13 op/s 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[99461]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: Reconfiguring mds.a (monmap changed)... 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: Reconfiguring daemon mds.a on smithi026 2024-09-13T14:23:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: pgmap v71: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 110 KiB/s wr, 13 op/s 2024-09-13T14:23:29.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:29 smithi026 ceph-mon[97022]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:23:29.921 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: Reconfiguring mds.a (monmap changed)... 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: Reconfiguring daemon mds.a on smithi026 2024-09-13T14:23:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: pgmap v71: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 110 KiB/s wr, 13 op/s 2024-09-13T14:23:29.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:29 smithi145 ceph-mon[89145]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 30s ago 6m 19.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (110s) 16s ago 6m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (8m) 30s ago 8m 14.7M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (8m) 16s ago 8m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (70s) 16s ago 11m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (99s) 30s ago 13m 516M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (57s) 30s ago 13m 40.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (21s) 16s ago 12m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:23:30.529 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (34s) 30s ago 12m 21.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 30s ago 6m 9.87M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (105s) 16s ago 6m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (11m) 30s ago 11m 163M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a db407fd1f6f4 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (11m) 30s ago 11m 151M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (10m) 30s ago 10m 153M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (10m) 30s ago 10m 88.7M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (9m) 16s ago 9m 171M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (9m) 16s ago 9m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (9m) 16s ago 9m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (8m) 16s ago 8m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:23:30.530 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (76s) 16s ago 7m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:23:30.561 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:30.883 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:23:30.883 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:23:30.884 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:23:30.885 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 10, 2024-09-13T14:23:30.885 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 5 2024-09-13T14:23:30.885 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:23:30.885 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:23:30.895 INFO:tasks.rados.rados.0.smithi026.stdout:1343: done (0 left) 2024-09-13T14:23:30.895 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 123 2024-09-13T14:23:30.895 INFO:tasks.rados.rados.0.smithi026.stdout:1344: seq_num 450 ranges {1310720=344064} 2024-09-13T14:23:30.904 INFO:tasks.rados.rados.0.smithi026.stdout:1344: writing smithi02683976-8 from 1310720 to 1654784 tid 1 2024-09-13T14:23:30.905 INFO:tasks.rados.rados.0.smithi026.stdout:1345: rollback oid 31 current snap is 123 2024-09-13T14:23:30.905 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 120 2024-09-13T14:23:30.905 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 123 2024-09-13T14:23:30.905 INFO:tasks.rados.rados.0.smithi026.stdout:1346: seq_num 451 ranges {8413184=360448} 2024-09-13T14:23:30.953 INFO:tasks.rados.rados.0.smithi026.stdout:1346: writing smithi02683976-20 from 8413184 to 8773632 tid 1 2024-09-13T14:23:30.953 INFO:tasks.rados.rados.0.smithi026.stdout:1347: read oid 41 snap -1 2024-09-13T14:23:30.953 INFO:tasks.rados.rados.0.smithi026.stdout:1347: expect deleted 2024-09-13T14:23:30.954 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 123 2024-09-13T14:23:30.954 INFO:tasks.rados.rados.0.smithi026.stdout:1345: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:23:30.954 INFO:tasks.rados.rados.0.smithi026.stdout:1348: seq_num 452 ranges {3424256=376832} 2024-09-13T14:23:30.975 INFO:tasks.rados.rados.0.smithi026.stdout:1348: writing smithi02683976-50 from 3424256 to 3801088 tid 1 2024-09-13T14:23:30.975 INFO:tasks.rados.rados.0.smithi026.stdout:1349: rollback oid 9 current snap is 123 2024-09-13T14:23:30.975 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 111 2024-09-13T14:23:30.975 INFO:tasks.rados.rados.0.smithi026.stdout:1345: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:23:30.975 INFO:tasks.rados.rados.0.smithi026.stdout:1345: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 753 (ObjNum 372 snap 104 seq_num 372) dirty exists 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1345: done (5 left) 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1350: read oid 11 snap -1 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1350: expect (ObjNum 422 snap 119 seq_num 422) 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1351: rollback oid 17 current snap is 123 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 111 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1344: finishing write tid 1 to smithi02683976-8 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1344: finishing write tid 2 to smithi02683976-8 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1344: finishing write tid 3 to smithi02683976-8 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 579 (ObjNum 450 snap 123 seq_num 450) dirty exists 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1344: left oid 8 (ObjNum 450 snap 123 seq_num 450) 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1344: done (6 left) 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1352: read oid 48 snap -1 2024-09-13T14:23:30.976 INFO:tasks.rados.rados.0.smithi026.stdout:1352: expect deleted 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1353: delete oid 24 current snap is 123 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1346: finishing write tid 1 to smithi02683976-20 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1346: finishing write tid 2 to smithi02683976-20 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1346: finishing write tid 3 to smithi02683976-20 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 688 (ObjNum 451 snap 123 seq_num 451) dirty exists 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1346: left oid 20 (ObjNum 451 snap 123 seq_num 451) 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:1349: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:23:30.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:23:30.978 INFO:tasks.rados.rados.0.smithi026.stdout:1346: done (7 left) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1347: done (6 left) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1349: done (5 left) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1352: done (4 left) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1353: done (3 left) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1354: rollback oid 45 current snap is 123 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 111 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1355: read oid 24 snap 111 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1355: expect deleted 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1356: read oid 3 snap -1 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1356: expect (ObjNum 420 snap 119 seq_num 420) 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 123 2024-09-13T14:23:30.979 INFO:tasks.rados.rados.0.smithi026.stdout:1357: seq_num 454 ranges {221184=409600} 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1357: writing smithi02683976-6 from 221184 to 630784 tid 1 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1358: rmattr oid 16 current snap is 123 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1358: done (7 left) 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1359: rmattr oid 42 current snap is 123 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1359: done (7 left) 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1360: snap_create 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1354: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 0 (ObjNum 33 snap 0 seq_num 509663312) dirty dne 2024-09-13T14:23:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1351: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:23:30.984 INFO:tasks.rados.rados.0.smithi026.stdout:1351: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:23:30.984 INFO:tasks.rados.rados.0.smithi026.stdout:1348: finishing write tid 1 to smithi02683976-50 2024-09-13T14:23:30.984 INFO:tasks.rados.rados.0.smithi026.stdout:1348: finishing write tid 2 to smithi02683976-50 2024-09-13T14:23:30.984 INFO:tasks.rados.rados.0.smithi026.stdout:1351: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:23:30.984 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 401 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:23:30.986 INFO:tasks.rados.rados.0.smithi026.stdout:1356: expect (ObjNum 420 snap 119 seq_num 420) 2024-09-13T14:23:31.046 INFO:tasks.rados.rados.0.smithi026.stdout:1348: finishing write tid 3 to smithi02683976-50 2024-09-13T14:23:31.046 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 755 (ObjNum 452 snap 123 seq_num 452) dirty exists 2024-09-13T14:23:31.047 INFO:tasks.rados.rados.0.smithi026.stdout:1348: left oid 50 (ObjNum 452 snap 123 seq_num 452) 2024-09-13T14:23:31.047 INFO:tasks.rados.rados.0.smithi026.stdout:1350: expect (ObjNum 422 snap 119 seq_num 422) 2024-09-13T14:23:31.106 INFO:tasks.rados.rados.0.smithi026.stdout:1357: finishing write tid 1 to smithi02683976-6 2024-09-13T14:23:31.106 INFO:tasks.rados.rados.0.smithi026.stdout:1357: finishing write tid 2 to smithi02683976-6 2024-09-13T14:23:31.106 INFO:tasks.rados.rados.0.smithi026.stdout:1357: finishing write tid 3 to smithi02683976-6 2024-09-13T14:23:31.106 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 757 (ObjNum 454 snap 123 seq_num 454) dirty exists 2024-09-13T14:23:31.106 INFO:tasks.rados.rados.0.smithi026.stdout:1357: left oid 6 (ObjNum 454 snap 123 seq_num 454) 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:31.146 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:23:31.147 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.147 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:30 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3198997286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:23:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3198997286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: Reconfiguring daemon mon.b on smithi145 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:31.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:30 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3198997286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:31.197 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:23:31.197 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "5/20 daemons upgraded", 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:23:31.198 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:23:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:31] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.51.0" 2024-09-13T14:23:31.907 INFO:tasks.rados.rados.0.smithi026.stdout:1348: done (7 left) 2024-09-13T14:23:31.907 INFO:tasks.rados.rados.0.smithi026.stdout:1350: done (6 left) 2024-09-13T14:23:31.907 INFO:tasks.rados.rados.0.smithi026.stdout:1351: done (5 left) 2024-09-13T14:23:31.907 INFO:tasks.rados.rados.0.smithi026.stdout:1354: done (4 left) 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1355: done (3 left) 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1356: done (2 left) 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1357: done (1 left) 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1360: done (0 left) 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1361: rollback oid 39 current snap is 124 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 120 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 124 2024-09-13T14:23:31.908 INFO:tasks.rados.rados.0.smithi026.stdout:1362: seq_num 455 ranges {5750784=425984} 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1362: writing smithi02683976-44 from 5750784 to 6176768 tid 1 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1363: read oid 16 snap 120 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1363: expect (ObjNum 405 snap 116 seq_num 405) 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1364: read oid 17 snap -1 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1364: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1365: read oid 34 snap 111 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1365: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1366: delete oid 49 current snap is 124 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1361: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1361: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:23:31.942 INFO:tasks.rados.rados.0.smithi026.stdout:1361: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:23:31.943 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 582 (ObjNum 408 snap 116 seq_num 408) dirty exists 2024-09-13T14:23:31.946 INFO:tasks.rados.rados.0.smithi026.stdout:1361: done (5 left) 2024-09-13T14:23:31.946 INFO:tasks.rados.rados.0.smithi026.stdout:1366: done (4 left) 2024-09-13T14:23:31.946 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 124 2024-09-13T14:23:31.946 INFO:tasks.rados.rados.0.smithi026.stdout:1367: seq_num 457 ranges {1941504=458752} 2024-09-13T14:23:31.962 INFO:tasks.rados.rados.0.smithi026.stdout:1367: writing smithi02683976-27 from 1941504 to 2400256 tid 1 2024-09-13T14:23:31.962 INFO:tasks.rados.rados.0.smithi026.stdout:1368: delete oid 11 current snap is 124 2024-09-13T14:23:31.962 INFO:tasks.rados.rados.0.smithi026.stdout:1362: finishing write tid 1 to smithi02683976-44 2024-09-13T14:23:31.962 INFO:tasks.rados.rados.0.smithi026.stdout:1362: finishing write tid 2 to smithi02683976-44 2024-09-13T14:23:31.962 INFO:tasks.rados.rados.0.smithi026.stdout:1362: finishing write tid 3 to smithi02683976-44 2024-09-13T14:23:31.963 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 437 (ObjNum 455 snap 124 seq_num 455) dirty exists 2024-09-13T14:23:31.963 INFO:tasks.rados.rados.0.smithi026.stdout:1362: left oid 44 (ObjNum 455 snap 124 seq_num 455) 2024-09-13T14:23:31.963 INFO:tasks.rados.rados.0.smithi026.stdout:1364: expect (ObjNum 326 snap 89 seq_num 326) 2024-09-13T14:23:32.011 INFO:tasks.rados.rados.0.smithi026.stdout:1363: expect (ObjNum 405 snap 116 seq_num 405) 2024-09-13T14:23:32.111 INFO:tasks.rados.rados.0.smithi026.stdout:1367: finishing write tid 1 to smithi02683976-27 2024-09-13T14:23:32.111 INFO:tasks.rados.rados.0.smithi026.stdout:1367: finishing write tid 2 to smithi02683976-27 2024-09-13T14:23:32.111 INFO:tasks.rados.rados.0.smithi026.stdout:1367: finishing write tid 3 to smithi02683976-27 2024-09-13T14:23:32.111 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 773 (ObjNum 457 snap 124 seq_num 457) dirty exists 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1367: left oid 27 (ObjNum 457 snap 124 seq_num 457) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1362: done (5 left) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1363: done (4 left) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1364: done (3 left) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1367: done (2 left) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1368: done (1 left) 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1369: rollback oid 42 current snap is 124 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 111 2024-09-13T14:23:32.112 INFO:tasks.rados.rados.0.smithi026.stdout:1365: expect (ObjNum 234 snap 58 seq_num 234) 2024-09-13T14:23:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: pgmap v74: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 111 KiB/s wr, 13 op/s 2024-09-13T14:23:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:23:32.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: pgmap v74: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 111 KiB/s wr, 13 op/s 2024-09-13T14:23:32.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: pgmap v74: 121 pgs: 121 active+clean; 206 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 111 KiB/s wr, 13 op/s 2024-09-13T14:23:32.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='client.54118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:23:32.168 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1369: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1369: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 310 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1365: done (1 left) 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1369: done (0 left) 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1370: read oid 30 snap -1 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1370: expect (ObjNum 366 snap 102 seq_num 366) 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1371: copy_from oid 6 from oid 41 current snap is 124 2024-09-13T14:23:32.702 INFO:tasks.rados.rados.0.smithi026.stdout:1372: snap_remove snap 120 2024-09-13T14:23:32.910 INFO:tasks.rados.rados.0.smithi026.stdout:1372: done (2 left) 2024-09-13T14:23:32.910 INFO:tasks.rados.rados.0.smithi026.stdout:1373: snap_remove snap 111 2024-09-13T14:23:33.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:23:33.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: Reconfiguring daemon mgr.x on smithi145 2024-09-13T14:23:33.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: Reconfiguring daemon mgr.x on smithi145 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:23:33.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:32 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:23:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: Reconfiguring daemon mgr.x on smithi145 2024-09-13T14:23:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:23:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:33.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:23:33.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:32 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1373: done (2 left) 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1374: copy_from oid 5 from oid 46 current snap is 124 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1371: finishing copy_from to smithi02683976-6 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1371: got expected ENOENT (src dne) 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1371: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:23:33.896 INFO:tasks.rados.rados.0.smithi026.stdout:1370: expect (ObjNum 366 snap 102 seq_num 366) 2024-09-13T14:23:33.947 INFO:tasks.rados.rados.0.smithi026.stdout:1374: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:23:33.947 INFO:tasks.rados.rados.0.smithi026.stdout:1374: finishing copy_from to smithi02683976-5 2024-09-13T14:23:33.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 427 (ObjNum 374 snap 104 seq_num 374) dirty exists 2024-09-13T14:23:33.948 INFO:tasks.rados.rados.0.smithi026.stdout:1370: done (2 left) 2024-09-13T14:23:33.948 INFO:tasks.rados.rados.0.smithi026.stdout:1371: done (1 left) 2024-09-13T14:23:33.948 INFO:tasks.rados.rados.0.smithi026.stdout:1374: done (0 left) 2024-09-13T14:23:33.948 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 124 2024-09-13T14:23:33.948 INFO:tasks.rados.rados.0.smithi026.stdout:1375: seq_num 459 ranges {3375104=491520} 2024-09-13T14:23:33.968 INFO:tasks.rados.rados.0.smithi026.stdout:1375: writing smithi02683976-38 from 3375104 to 3866624 tid 1 2024-09-13T14:23:33.969 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 124 2024-09-13T14:23:33.969 INFO:tasks.rados.rados.0.smithi026.stdout:1376: seq_num 460 ranges {8486912=507904} 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1376: writing smithi02683976-34 from 8486912 to 8994816 tid 1 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 124 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1375: finishing write tid 1 to smithi02683976-38 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1375: finishing write tid 2 to smithi02683976-38 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1375: finishing write tid 3 to smithi02683976-38 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 658 (ObjNum 459 snap 124 seq_num 459) dirty exists 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1375: left oid 38 (ObjNum 459 snap 124 seq_num 459) 2024-09-13T14:23:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:1377: seq_num 461 ranges {7774208=524288} 2024-09-13T14:23:34.063 INFO:tasks.rados.rados.0.smithi026.stdout:1377: writing smithi02683976-22 from 7774208 to 8298496 tid 1 2024-09-13T14:23:34.063 INFO:tasks.rados.rados.0.smithi026.stdout:1375: done (2 left) 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1378: rmattr oid 15 current snap is 124 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1378: done (2 left) 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1379: delete oid 20 current snap is 124 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1376: finishing write tid 1 to smithi02683976-34 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1376: finishing write tid 2 to smithi02683976-34 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1376: finishing write tid 3 to smithi02683976-34 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 403 (ObjNum 460 snap 124 seq_num 460) dirty exists 2024-09-13T14:23:34.064 INFO:tasks.rados.rados.0.smithi026.stdout:1376: left oid 34 (ObjNum 460 snap 124 seq_num 460) 2024-09-13T14:23:34.065 INFO:tasks.rados.rados.0.smithi026.stdout:1376: done (2 left) 2024-09-13T14:23:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:1379: done (1 left) 2024-09-13T14:23:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 124 2024-09-13T14:23:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:1380: seq_num 463 ranges {1474560=557056} 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1380: writing smithi02683976-5 from 1474560 to 2031616 tid 1 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1381: delete oid 38 current snap is 124 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1377: finishing write tid 1 to smithi02683976-22 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1377: finishing write tid 2 to smithi02683976-22 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1377: finishing write tid 3 to smithi02683976-22 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 593 (ObjNum 461 snap 124 seq_num 461) dirty exists 2024-09-13T14:23:34.077 INFO:tasks.rados.rados.0.smithi026.stdout:1377: left oid 22 (ObjNum 461 snap 124 seq_num 461) 2024-09-13T14:23:34.078 INFO:tasks.rados.rados.0.smithi026.stdout:1377: done (2 left) 2024-09-13T14:23:34.078 INFO:tasks.rados.rados.0.smithi026.stdout:1381: done (1 left) 2024-09-13T14:23:34.079 INFO:tasks.rados.rados.0.smithi026.stdout:1382: setattr oid 17 current snap is 124 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:1380: finishing write tid 1 to smithi02683976-5 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:1380: finishing write tid 2 to smithi02683976-5 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:1383: read oid 21 snap -1 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:1383: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 124 2024-09-13T14:23:34.080 INFO:tasks.rados.rados.0.smithi026.stdout:1384: seq_num 465 ranges {6905856=598016} 2024-09-13T14:23:34.121 INFO:tasks.rados.rados.0.smithi026.stdout:1384: writing smithi02683976-26 from 6905856 to 7503872 tid 1 2024-09-13T14:23:34.121 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 124 2024-09-13T14:23:34.121 INFO:tasks.rados.rados.0.smithi026.stdout:1380: finishing write tid 3 to smithi02683976-5 2024-09-13T14:23:34.122 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 793 (ObjNum 463 snap 124 seq_num 463) dirty exists 2024-09-13T14:23:34.122 INFO:tasks.rados.rados.0.smithi026.stdout:1380: left oid 5 (ObjNum 463 snap 124 seq_num 463) 2024-09-13T14:23:34.122 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 404 (ObjNum 326 snap 89 seq_num 326) dirty exists 2024-09-13T14:23:34.122 INFO:tasks.rados.rados.0.smithi026.stdout:1383: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:23:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[97022]: pgmap v77: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[97022]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:23:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.4 on smithi145 2024-09-13T14:23:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[97022]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:23:34.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[97022]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:23:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[99461]: pgmap v77: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[99461]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:23:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.4 on smithi145 2024-09-13T14:23:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[99461]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:23:34.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:33 smithi026 ceph-mon[99461]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:23:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:33 smithi145 ceph-mon[89145]: pgmap v77: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:33 smithi145 ceph-mon[89145]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:23:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:33 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.4 on smithi145 2024-09-13T14:23:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:33 smithi145 ceph-mon[89145]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:23:34.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:33 smithi145 ceph-mon[89145]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:23:34.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:34.291 INFO:tasks.rados.rados.0.smithi026.stdout:1384: finishing write tid 1 to smithi02683976-26 2024-09-13T14:23:34.291 INFO:tasks.rados.rados.0.smithi026.stdout:1384: finishing write tid 2 to smithi02683976-26 2024-09-13T14:23:34.291 INFO:tasks.rados.rados.0.smithi026.stdout:1384: finishing write tid 3 to smithi02683976-26 2024-09-13T14:23:34.292 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 595 (ObjNum 465 snap 124 seq_num 465) dirty exists 2024-09-13T14:23:34.292 INFO:tasks.rados.rados.0.smithi026.stdout:1384: left oid 26 (ObjNum 465 snap 124 seq_num 465) 2024-09-13T14:23:34.292 INFO:tasks.rados.rados.0.smithi026.stdout:1385: seq_num 466 ranges {2228224=614400} 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1385: writing smithi02683976-7 from 2228224 to 2842624 tid 1 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1380: done (4 left) 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1382: done (3 left) 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1383: done (2 left) 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1384: done (1 left) 2024-09-13T14:23:34.307 INFO:tasks.rados.rados.0.smithi026.stdout:1386: rollback oid 22 current snap is 124 2024-09-13T14:23:34.308 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 123 2024-09-13T14:23:34.308 INFO:tasks.rados.rados.0.smithi026.stdout:1387: rollback oid 47 current snap is 124 2024-09-13T14:23:34.308 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 123 2024-09-13T14:23:34.308 INFO:tasks.rados.rados.0.smithi026.stdout:1388: snap_create 2024-09-13T14:23:34.309 INFO:tasks.rados.rados.0.smithi026.stdout:1387: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:23:34.310 INFO:tasks.rados.rados.0.smithi026.stdout:1387: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:23:34.310 INFO:tasks.rados.rados.0.smithi026.stdout:1386: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:23:34.310 INFO:tasks.rados.rados.0.smithi026.stdout:1385: finishing write tid 1 to smithi02683976-7 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1387: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 296 (ObjNum 438 snap 122 seq_num 438) dirty exists 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1386: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1385: finishing write tid 2 to smithi02683976-7 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1386: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 598 (ObjNum 380 snap 106 seq_num 380) dirty exists 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1385: finishing write tid 3 to smithi02683976-7 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 510 (ObjNum 466 snap 124 seq_num 466) dirty exists 2024-09-13T14:23:34.311 INFO:tasks.rados.rados.0.smithi026.stdout:1385: left oid 7 (ObjNum 466 snap 124 seq_num 466) 2024-09-13T14:23:34.914 INFO:tasks.rados.rados.0.smithi026.stdout:1385: done (3 left) 2024-09-13T14:23:34.915 INFO:tasks.rados.rados.0.smithi026.stdout:1386: done (2 left) 2024-09-13T14:23:34.915 INFO:tasks.rados.rados.0.smithi026.stdout:1387: done (1 left) 2024-09-13T14:23:34.915 INFO:tasks.rados.rados.0.smithi026.stdout:1388: done (0 left) 2024-09-13T14:23:34.915 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 125 2024-09-13T14:23:34.915 INFO:tasks.rados.rados.0.smithi026.stdout:1389: seq_num 467 ranges {2031616=630784} 2024-09-13T14:23:34.929 INFO:tasks.rados.rados.0.smithi026.stdout:1389: writing smithi02683976-5 from 2031616 to 2662400 tid 1 2024-09-13T14:23:34.929 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 125 2024-09-13T14:23:34.929 INFO:tasks.rados.rados.0.smithi026.stdout:1390: seq_num 468 ranges {6389760=647168} 2024-09-13T14:23:34.968 INFO:tasks.rados.rados.0.smithi026.stdout:1390: writing smithi02683976-10 from 6389760 to 7036928 tid 1 2024-09-13T14:23:34.968 INFO:tasks.rados.rados.0.smithi026.stdout:1391: rmattr oid 30 current snap is 125 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1389: finishing write tid 1 to smithi02683976-5 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1389: finishing write tid 2 to smithi02683976-5 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1389: finishing write tid 3 to smithi02683976-5 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 796 (ObjNum 467 snap 125 seq_num 467) dirty exists 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1389: left oid 5 (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1389: done (2 left) 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1392: copy_from oid 34 from oid 50 current snap is 125 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1393: read oid 28 snap -1 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1393: expect (ObjNum 307 snap 83 seq_num 307) 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 125 2024-09-13T14:23:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:1394: seq_num 469 ranges {7774208=663552} 2024-09-13T14:23:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1394: writing smithi02683976-22 from 7774208 to 8437760 tid 1 2024-09-13T14:23:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1395: copy_from oid 42 from oid 25 current snap is 125 2024-09-13T14:23:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 576 (ObjNum 366 snap 102 seq_num 366) dirty exists 2024-09-13T14:23:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1391: done (5 left) 2024-09-13T14:23:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1396: snap_remove snap 124 2024-09-13T14:23:35.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[97022]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:23:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:35.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:35 smithi026 ceph-mon[99461]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:23:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:23:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-mon[89145]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:23:35.918 INFO:tasks.rados.rados.0.smithi026.stdout:1396: done (5 left) 2024-09-13T14:23:35.918 INFO:tasks.rados.rados.0.smithi026.stdout:1397: snap_remove snap 123 2024-09-13T14:23:36.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:36.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:36.576 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:23:36.576 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.5 on smithi145 2024-09-13T14:23:36.576 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: pgmap v80: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:36.576 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.577 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.577 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:23:36.577 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:36.577 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-mon[89145]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:23:36.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:23:36.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.5 on smithi145 2024-09-13T14:23:36.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: pgmap v80: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:36.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[99461]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.5 on smithi145 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: pgmap v80: 121 pgs: 121 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.2 MiB/s wr, 30 op/s 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:23:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:36.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:36 smithi026 ceph-mon[97022]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1397: done (5 left) 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 125 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1390: finishing write tid 1 to smithi02683976-10 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1390: finishing write tid 2 to smithi02683976-10 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1392: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1392: finishing copy_from to smithi02683976-34 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 755 (ObjNum 452 snap 123 seq_num 452) dirty exists 2024-09-13T14:23:36.919 INFO:tasks.rados.rados.0.smithi026.stdout:1393: expect (ObjNum 307 snap 83 seq_num 307) 2024-09-13T14:23:37.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:36 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:37.224 INFO:tasks.rados.rados.0.smithi026.stdout:1390: finishing write tid 3 to smithi02683976-10 2024-09-13T14:23:37.224 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 435 (ObjNum 468 snap 125 seq_num 468) dirty exists 2024-09-13T14:23:37.224 INFO:tasks.rados.rados.0.smithi026.stdout:1390: left oid 10 (ObjNum 468 snap 125 seq_num 468) 2024-09-13T14:23:37.224 INFO:tasks.rados.rados.0.smithi026.stdout:1394: finishing write tid 1 to smithi02683976-22 2024-09-13T14:23:37.224 INFO:tasks.rados.rados.0.smithi026.stdout:1394: finishing write tid 2 to smithi02683976-22 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:1394: finishing write tid 3 to smithi02683976-22 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 600 (ObjNum 469 snap 125 seq_num 469) dirty exists 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:1394: left oid 22 (ObjNum 469 snap 125 seq_num 469) 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:1395: finishing copy_from racing read to smithi02683976-42 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:1395: finishing copy_from to smithi02683976-42 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 360 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:23:37.225 INFO:tasks.rados.rados.0.smithi026.stdout:1398: seq_num 470 ranges {2244608=679936} 2024-09-13T14:23:37.240 INFO:tasks.rados.rados.0.smithi026.stdout:1398: writing smithi02683976-3 from 2244608 to 2924544 tid 1 2024-09-13T14:23:37.240 INFO:tasks.rados.rados.0.smithi026.stdout:1390: done (5 left) 2024-09-13T14:23:37.240 INFO:tasks.rados.rados.0.smithi026.stdout:1392: done (4 left) 2024-09-13T14:23:37.241 INFO:tasks.rados.rados.0.smithi026.stdout:1393: done (3 left) 2024-09-13T14:23:37.241 INFO:tasks.rados.rados.0.smithi026.stdout:1394: done (2 left) 2024-09-13T14:23:37.241 INFO:tasks.rados.rados.0.smithi026.stdout:1395: done (1 left) 2024-09-13T14:23:37.241 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 125 2024-09-13T14:23:37.241 INFO:tasks.rados.rados.0.smithi026.stdout:1399: seq_num 471 ranges {1335296=696320} 2024-09-13T14:23:37.252 INFO:tasks.rados.rados.0.smithi026.stdout:1399: writing smithi02683976-30 from 1335296 to 2031616 tid 1 2024-09-13T14:23:37.252 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 125 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:1398: finishing write tid 1 to smithi02683976-3 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:1398: finishing write tid 2 to smithi02683976-3 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:1398: finishing write tid 3 to smithi02683976-3 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 800 (ObjNum 470 snap 125 seq_num 470) dirty exists 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:1398: left oid 3 (ObjNum 470 snap 125 seq_num 470) 2024-09-13T14:23:37.253 INFO:tasks.rados.rados.0.smithi026.stdout:1400: seq_num 472 ranges {0=712704} 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1400: writing smithi02683976-48 from 0 to 712704 tid 1 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1398: done (2 left) 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1401: copy_from oid 6 from oid 42 current snap is 125 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1399: finishing write tid 1 to smithi02683976-30 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1399: finishing write tid 2 to smithi02683976-30 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1402: copy_from oid 2 from oid 14 current snap is 125 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1403: rollback oid 5 current snap is 125 2024-09-13T14:23:37.258 INFO:tasks.rados.rados.0.smithi026.stdout:1403: done (4 left) 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1404: read oid 40 snap -1 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1404: expect (ObjNum 440 snap 122 seq_num 440) 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1405: setattr oid 12 current snap is 125 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1399: finishing write tid 3 to smithi02683976-30 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 578 (ObjNum 471 snap 125 seq_num 471) dirty exists 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1399: left oid 30 (ObjNum 471 snap 125 seq_num 471) 2024-09-13T14:23:37.259 INFO:tasks.rados.rados.0.smithi026.stdout:1399: done (5 left) 2024-09-13T14:23:37.260 INFO:tasks.rados.rados.0.smithi026.stdout:1406: delete oid 29 current snap is 125 2024-09-13T14:23:37.260 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 601 (ObjNum 389 snap 110 seq_num 389) dirty exists 2024-09-13T14:23:37.261 INFO:tasks.rados.rados.0.smithi026.stdout:1400: finishing write tid 1 to smithi02683976-48 2024-09-13T14:23:37.261 INFO:tasks.rados.rados.0.smithi026.stdout:1405: done (5 left) 2024-09-13T14:23:37.261 INFO:tasks.rados.rados.0.smithi026.stdout:1406: done (4 left) 2024-09-13T14:23:37.261 INFO:tasks.rados.rados.0.smithi026.stdout:1407: snap_create 2024-09-13T14:23:37.265 INFO:tasks.rados.rados.0.smithi026.stdout:1400: finishing write tid 2 to smithi02683976-48 2024-09-13T14:23:37.266 INFO:tasks.rados.rados.0.smithi026.stdout:1400: finishing write tid 3 to smithi02683976-48 2024-09-13T14:23:37.266 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 298 (ObjNum 472 snap 125 seq_num 472) dirty exists 2024-09-13T14:23:37.266 INFO:tasks.rados.rados.0.smithi026.stdout:1400: left oid 48 (ObjNum 472 snap 125 seq_num 472) 2024-09-13T14:23:37.279 INFO:tasks.rados.rados.0.smithi026.stdout:1404: expect (ObjNum 440 snap 122 seq_num 440) 2024-09-13T14:23:37.545 INFO:tasks.rados.rados.0.smithi026.stdout:1401: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:23:37.545 INFO:tasks.rados.rados.0.smithi026.stdout:1402: finishing copy_from racing read to smithi02683976-2 2024-09-13T14:23:37.545 INFO:tasks.rados.rados.0.smithi026.stdout:1401: finishing copy_from to smithi02683976-6 2024-09-13T14:23:37.546 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 360 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:23:37.546 INFO:tasks.rados.rados.0.smithi026.stdout:1402: finishing copy_from to smithi02683976-2 2024-09-13T14:23:37.546 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 594 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:23:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[99461]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:23:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.6 on smithi145 2024-09-13T14:23:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[99461]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:23:37.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[97022]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:23:37.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.6 on smithi145 2024-09-13T14:23:37.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:37 smithi026 ceph-mon[97022]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:23:37.697 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:37 smithi145 ceph-mon[89145]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:23:37.697 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:37 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.6 on smithi145 2024-09-13T14:23:37.697 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:37 smithi145 ceph-mon[89145]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:23:37.919 INFO:tasks.rados.rados.0.smithi026.stdout:1400: done (4 left) 2024-09-13T14:23:37.919 INFO:tasks.rados.rados.0.smithi026.stdout:1401: done (3 left) 2024-09-13T14:23:37.919 INFO:tasks.rados.rados.0.smithi026.stdout:1402: done (2 left) 2024-09-13T14:23:37.919 INFO:tasks.rados.rados.0.smithi026.stdout:1404: done (1 left) 2024-09-13T14:23:37.919 INFO:tasks.rados.rados.0.smithi026.stdout:1407: done (0 left) 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1408: rmattr oid 47 current snap is 126 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1408: done (0 left) 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1409: read oid 24 snap 125 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1409: expect deleted 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1410: read oid 41 snap -1 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1410: expect deleted 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1411: read oid 7 snap -1 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1411: expect (ObjNum 466 snap 124 seq_num 466) 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1412: rmattr oid 22 current snap is 126 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1412: done (3 left) 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1413: copy_from oid 49 from oid 44 current snap is 126 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 126 2024-09-13T14:23:37.920 INFO:tasks.rados.rados.0.smithi026.stdout:1414: seq_num 474 ranges {7774208=745472} 2024-09-13T14:23:37.966 INFO:tasks.rados.rados.0.smithi026.stdout:1414: writing smithi02683976-35 from 7774208 to 8519680 tid 1 2024-09-13T14:23:37.966 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 126 2024-09-13T14:23:37.966 INFO:tasks.rados.rados.0.smithi026.stdout:1415: seq_num 475 ranges {0=761856} 2024-09-13T14:23:37.970 INFO:tasks.rados.rados.0.smithi026.stdout:1415: writing smithi02683976-1 from 0 to 761856 tid 1 2024-09-13T14:23:37.970 INFO:tasks.rados.rados.0.smithi026.stdout:1416: setattr oid 48 current snap is 126 2024-09-13T14:23:37.970 INFO:tasks.rados.rados.0.smithi026.stdout:1411: expect (ObjNum 466 snap 124 seq_num 466) 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1413: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1413: finishing copy_from to smithi02683976-49 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 437 (ObjNum 455 snap 124 seq_num 455) dirty exists 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1414: finishing write tid 1 to smithi02683976-35 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1414: finishing write tid 2 to smithi02683976-35 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1414: finishing write tid 3 to smithi02683976-35 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 807 (ObjNum 474 snap 126 seq_num 474) dirty exists 2024-09-13T14:23:38.083 INFO:tasks.rados.rados.0.smithi026.stdout:1414: left oid 35 (ObjNum 474 snap 126 seq_num 474) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1415: finishing write tid 1 to smithi02683976-1 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1415: finishing write tid 2 to smithi02683976-1 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1415: finishing write tid 3 to smithi02683976-1 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 809 (ObjNum 475 snap 126 seq_num 475) dirty exists 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1415: left oid 1 (ObjNum 475 snap 126 seq_num 475) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1409: done (6 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1410: done (5 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1411: done (4 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1413: done (3 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1414: done (2 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1415: done (1 left) 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:1417: rollback oid 2 current snap is 126 2024-09-13T14:23:38.084 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 125 2024-09-13T14:23:38.085 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 126 2024-09-13T14:23:38.085 INFO:tasks.rados.rados.0.smithi026.stdout:1418: seq_num 476 ranges {1982464=778240} 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1418: writing smithi02683976-47 from 1982464 to 2760704 tid 1 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1419: rmattr oid 16 current snap is 126 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1419: done (3 left) 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1420: read oid 9 snap -1 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1420: expect deleted 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:1421: snap_create 2024-09-13T14:23:38.098 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 299 (ObjNum 472 snap 125 seq_num 472) dirty exists 2024-09-13T14:23:38.099 INFO:tasks.rados.rados.0.smithi026.stdout:1417: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:23:38.099 INFO:tasks.rados.rados.0.smithi026.stdout:1417: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:23:38.099 INFO:tasks.rados.rados.0.smithi026.stdout:1417: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:23:38.099 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 597 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:23:38.100 INFO:tasks.rados.rados.0.smithi026.stdout:1418: finishing write tid 1 to smithi02683976-47 2024-09-13T14:23:38.100 INFO:tasks.rados.rados.0.smithi026.stdout:1418: finishing write tid 2 to smithi02683976-47 2024-09-13T14:23:38.101 INFO:tasks.rados.rados.0.smithi026.stdout:1418: finishing write tid 3 to smithi02683976-47 2024-09-13T14:23:38.101 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 301 (ObjNum 476 snap 126 seq_num 476) dirty exists 2024-09-13T14:23:38.101 INFO:tasks.rados.rados.0.smithi026.stdout:1418: left oid 47 (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:23:38.647 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: pgmap v83: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:38.647 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: Reconfiguring daemon osd.7 on smithi145 2024-09-13T14:23:38.648 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:38 smithi145 ceph-mon[89145]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:23:38.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: pgmap v83: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: Reconfiguring daemon osd.7 on smithi145 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[97022]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: pgmap v83: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:23:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:23:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: Reconfiguring daemon osd.7 on smithi145 2024-09-13T14:23:38.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:38 smithi026 ceph-mon[99461]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:23:38.896 INFO:tasks.rados.rados.0.smithi026.stdout:1416: done (4 left) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1417: done (3 left) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1418: done (2 left) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1420: done (1 left) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1421: done (0 left) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1422: read oid 49 snap -1 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1422: expect (ObjNum 455 snap 124 seq_num 455) 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 127 2024-09-13T14:23:38.897 INFO:tasks.rados.rados.0.smithi026.stdout:1423: seq_num 477 ranges {7503872=794624} 2024-09-13T14:23:38.942 INFO:tasks.rados.rados.0.smithi026.stdout:1423: writing smithi02683976-26 from 7503872 to 8298496 tid 1 2024-09-13T14:23:38.942 INFO:tasks.rados.rados.0.smithi026.stdout:1424: snap_create 2024-09-13T14:23:38.942 INFO:tasks.rados.rados.0.smithi026.stdout:1422: expect (ObjNum 455 snap 124 seq_num 455) 2024-09-13T14:23:39.348 INFO:tasks.rados.rados.0.smithi026.stdout:1423: finishing write tid 1 to smithi02683976-26 2024-09-13T14:23:39.348 INFO:tasks.rados.rados.0.smithi026.stdout:1423: finishing write tid 2 to smithi02683976-26 2024-09-13T14:23:39.349 INFO:tasks.rados.rados.0.smithi026.stdout:1423: finishing write tid 3 to smithi02683976-26 2024-09-13T14:23:39.349 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 603 (ObjNum 477 snap 127 seq_num 477) dirty exists 2024-09-13T14:23:39.349 INFO:tasks.rados.rados.0.smithi026.stdout:1423: left oid 26 (ObjNum 477 snap 127 seq_num 477) 2024-09-13T14:23:39.900 INFO:tasks.rados.rados.0.smithi026.stdout:1422: done (2 left) 2024-09-13T14:23:39.900 INFO:tasks.rados.rados.0.smithi026.stdout:1423: done (1 left) 2024-09-13T14:23:39.900 INFO:tasks.rados.rados.0.smithi026.stdout:1424: done (0 left) 2024-09-13T14:23:39.901 INFO:tasks.rados.rados.0.smithi026.stdout:1425: snap_create 2024-09-13T14:23:40.079 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: pgmap v86: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: Reconfiguring mds.b (monmap changed)... 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:40.080 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:39 smithi145 ceph-mon[89145]: Reconfiguring daemon mds.b on smithi145 2024-09-13T14:23:40.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: pgmap v86: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: Reconfiguring mds.b (monmap changed)... 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[97022]: Reconfiguring daemon mds.b on smithi145 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: pgmap v86: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.7 MiB/s wr, 25 op/s 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: Reconfiguring mds.b (monmap changed)... 2024-09-13T14:23:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:23:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:39 smithi026 ceph-mon[99461]: Reconfiguring daemon mds.b on smithi145 2024-09-13T14:23:40.380 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:40.902 INFO:tasks.rados.rados.0.smithi026.stdout:1425: done (0 left) 2024-09-13T14:23:40.902 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 129 2024-09-13T14:23:40.902 INFO:tasks.rados.rados.0.smithi026.stdout:1426: seq_num 478 ranges {6176768=409600,6586368=401408} 2024-09-13T14:23:40.938 INFO:tasks.rados.rados.0.smithi026.stdout:1426: writing smithi02683976-44 from 6176768 to 6586368 tid 1 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1426: writing smithi02683976-44 from 6586368 to 6987776 tid 2 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1427: read oid 10 snap -1 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1427: expect (ObjNum 468 snap 125 seq_num 468) 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1428: read oid 1 snap -1 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1428: expect (ObjNum 475 snap 126 seq_num 475) 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 129 2024-09-13T14:23:40.941 INFO:tasks.rados.rados.0.smithi026.stdout:1429: seq_num 479 ranges {0=425984,425984=401408} 2024-09-13T14:23:40.943 INFO:tasks.rados.rados.0.smithi026.stdout:1429: writing smithi02683976-19 from 0 to 425984 tid 1 2024-09-13T14:23:40.946 INFO:tasks.rados.rados.0.smithi026.stdout:1429: writing smithi02683976-19 from 425984 to 827392 tid 2 2024-09-13T14:23:40.946 INFO:tasks.rados.rados.0.smithi026.stdout:1430: copy_from oid 17 from oid 18 current snap is 129 2024-09-13T14:23:40.946 INFO:tasks.rados.rados.0.smithi026.stdout:1426: finishing write tid 1 to smithi02683976-44 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:1426: finishing write tid 2 to smithi02683976-44 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:1426: finishing write tid 3 to smithi02683976-44 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:1431: snap_create 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:1426: finishing write tid 4 to smithi02683976-44 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 440 (ObjNum 478 snap 129 seq_num 478) dirty exists 2024-09-13T14:23:40.947 INFO:tasks.rados.rados.0.smithi026.stdout:1426: left oid 44 (ObjNum 478 snap 129 seq_num 478) 2024-09-13T14:23:40.948 INFO:tasks.rados.rados.0.smithi026.stdout:1428: expect (ObjNum 475 snap 126 seq_num 475) 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1429: finishing write tid 1 to smithi02683976-19 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1429: finishing write tid 2 to smithi02683976-19 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1429: finishing write tid 3 to smithi02683976-19 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1429: finishing write tid 4 to smithi02683976-19 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 706 (ObjNum 479 snap 129 seq_num 479) dirty exists 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1429: left oid 19 (ObjNum 479 snap 129 seq_num 479) 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1430: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:23:40.976 INFO:tasks.rados.rados.0.smithi026.stdout:1430: finishing copy_from to smithi02683976-17 2024-09-13T14:23:40.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 589 (ObjNum 433 snap 122 seq_num 433) dirty exists 2024-09-13T14:23:40.977 INFO:tasks.rados.rados.0.smithi026.stdout:1427: expect (ObjNum 468 snap 125 seq_num 468) 2024-09-13T14:23:41.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:23:41.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:41.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:23:41.599 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:41] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.51.0" 2024-09-13T14:23:41.912 INFO:tasks.rados.rados.0.smithi026.stdout:1426: done (5 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1427: done (4 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1428: done (3 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1429: done (2 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1430: done (1 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1431: done (0 left) 2024-09-13T14:23:41.913 INFO:tasks.rados.rados.0.smithi026.stdout:1432: snap_create 2024-09-13T14:23:42.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all mon 2024-09-13T14:23:42.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all crash 2024-09-13T14:23:42.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: Upgrade: osd.0 is safe to restart 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: pgmap v89: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all mon 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all crash 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: Upgrade: osd.0 is safe to restart 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: pgmap v89: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:23:42.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all mon 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all crash 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: Upgrade: osd.0 is safe to restart 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: pgmap v89: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:23:42.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:42.914 INFO:tasks.rados.rados.0.smithi026.stdout:1432: done (0 left) 2024-09-13T14:23:42.914 INFO:tasks.rados.rados.0.smithi026.stdout:1433: snap_remove snap 128 2024-09-13T14:23:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:23:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: Upgrade: Updating osd.0 2024-09-13T14:23:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:43.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:43.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:42 smithi145 ceph-mon[89145]: Deploying daemon osd.0 on smithi026 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: Upgrade: Updating osd.0 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[99461]: Deploying daemon osd.0 on smithi026 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: Upgrade: Updating osd.0 2024-09-13T14:23:43.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:43.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:23:43.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:43.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:42 smithi026 ceph-mon[97022]: Deploying daemon osd.0 on smithi026 2024-09-13T14:23:43.897 INFO:tasks.rados.rados.0.smithi026.stdout:1433: done (0 left) 2024-09-13T14:23:43.898 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 131 2024-09-13T14:23:43.898 INFO:tasks.rados.rados.0.smithi026.stdout:1434: seq_num 480 ranges {0=442368,442368=401408} 2024-09-13T14:23:43.900 INFO:tasks.rados.rados.0.smithi026.stdout:1434: writing smithi02683976-33 from 0 to 442368 tid 1 2024-09-13T14:23:43.903 INFO:tasks.rados.rados.0.smithi026.stdout:1434: writing smithi02683976-33 from 442368 to 843776 tid 2 2024-09-13T14:23:43.903 INFO:tasks.rados.rados.0.smithi026.stdout:1435: copy_from oid 22 from oid 50 current snap is 131 2024-09-13T14:23:43.903 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 131 2024-09-13T14:23:43.903 INFO:tasks.rados.rados.0.smithi026.stdout:1436: seq_num 481 ranges {0=458752,458752=401408} 2024-09-13T14:23:43.906 INFO:tasks.rados.rados.0.smithi026.stdout:1436: writing smithi02683976-38 from 0 to 458752 tid 1 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1436: writing smithi02683976-38 from 458752 to 860160 tid 2 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1434: finishing write tid 1 to smithi02683976-33 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1437: read oid 17 snap -1 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1437: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1438: read oid 9 snap -1 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1438: expect deleted 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 131 2024-09-13T14:23:43.910 INFO:tasks.rados.rados.0.smithi026.stdout:1439: seq_num 482 ranges {2400256=475136,2875392=401408} 2024-09-13T14:23:43.926 INFO:tasks.rados.rados.0.smithi026.stdout:1439: writing smithi02683976-27 from 2400256 to 2875392 tid 1 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:1439: writing smithi02683976-27 from 2875392 to 3276800 tid 2 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:1440: setattr oid 10 current snap is 131 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:1434: finishing write tid 2 to smithi02683976-33 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:1434: finishing write tid 3 to smithi02683976-33 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:1434: finishing write tid 4 to smithi02683976-33 2024-09-13T14:23:43.929 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 813 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1434: left oid 33 (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1436: finishing write tid 1 to smithi02683976-38 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1436: finishing write tid 2 to smithi02683976-38 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1436: finishing write tid 3 to smithi02683976-38 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1436: finishing write tid 4 to smithi02683976-38 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 664 (ObjNum 481 snap 131 seq_num 481) dirty exists 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1436: left oid 38 (ObjNum 481 snap 131 seq_num 481) 2024-09-13T14:23:43.930 INFO:tasks.rados.rados.0.smithi026.stdout:1435: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:1434: done (6 left) 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:1436: done (5 left) 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:1438: done (4 left) 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:1441: snap_create 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:1435: finishing copy_from to smithi02683976-22 2024-09-13T14:23:43.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 755 (ObjNum 452 snap 123 seq_num 452) dirty exists 2024-09-13T14:23:43.932 INFO:tasks.rados.rados.0.smithi026.stdout:1439: finishing write tid 1 to smithi02683976-27 2024-09-13T14:23:43.932 INFO:tasks.rados.rados.0.smithi026.stdout:1439: finishing write tid 2 to smithi02683976-27 2024-09-13T14:23:43.932 INFO:tasks.rados.rados.0.smithi026.stdout:1437: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:23:44.196 INFO:tasks.rados.rados.0.smithi026.stdout:1439: finishing write tid 3 to smithi02683976-27 2024-09-13T14:23:44.196 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 444 (ObjNum 468 snap 125 seq_num 468) dirty exists 2024-09-13T14:23:44.196 INFO:tasks.rados.rados.0.smithi026.stdout:1439: finishing write tid 4 to smithi02683976-27 2024-09-13T14:23:44.197 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 817 (ObjNum 482 snap 131 seq_num 482) dirty exists 2024-09-13T14:23:44.197 INFO:tasks.rados.rados.0.smithi026.stdout:1439: left oid 27 (ObjNum 482 snap 131 seq_num 482) 2024-09-13T14:23:44.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:43 smithi145 ceph-mon[89145]: pgmap v92: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:44.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:43 smithi145 ceph-mon[89145]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:23:44.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:43 smithi145 ceph-mon[89145]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:23:44.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:44.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:44.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[99461]: pgmap v92: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:44.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[99461]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:23:44.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[99461]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:23:44.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[97022]: pgmap v92: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:44.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[97022]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:23:44.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:43 smithi026 ceph-mon[97022]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:23:44.906 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:44 smithi026 systemd[1]: Stopping Ceph osd.0 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:1435: done (4 left) 2024-09-13T14:23:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:1437: done (3 left) 2024-09-13T14:23:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:1439: done (2 left) 2024-09-13T14:23:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:1440: done (1 left) 2024-09-13T14:23:44.925 INFO:tasks.rados.rados.0.smithi026.stdout:1441: done (0 left) 2024-09-13T14:23:44.926 INFO:tasks.rados.rados.0.smithi026.stdout:1442: rmattr oid 41 current snap is 132 2024-09-13T14:23:44.926 INFO:tasks.rados.rados.0.smithi026.stdout:1442: done (0 left) 2024-09-13T14:23:44.926 INFO:tasks.rados.rados.0.smithi026.stdout:1443: snap_remove snap 126 2024-09-13T14:23:45.665 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[43451]: 2024-09-13T14:23:45.178+0000 7f410817c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:23:45.665 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[43451]: 2024-09-13T14:23:45.178+0000 7f410817c640 -1 osd.0 332 *** Got signal Terminated *** 2024-09-13T14:23:45.665 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[43451]: 2024-09-13T14:23:45.178+0000 7f410817c640 -1 osd.0 332 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:23:45.928 INFO:tasks.rados.rados.0.smithi026.stdout:1443: done (0 left) 2024-09-13T14:23:45.928 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 132 2024-09-13T14:23:45.928 INFO:tasks.rados.rados.0.smithi026.stdout:1444: seq_num 483 ranges {0=491520,491520=401408} 2024-09-13T14:23:45.930 INFO:tasks.rados.rados.0.smithi026.stdout:1444: writing smithi02683976-45 from 0 to 491520 tid 1 2024-09-13T14:23:45.933 INFO:tasks.rados.rados.0.smithi026.stdout:1444: writing smithi02683976-45 from 491520 to 892928 tid 2 2024-09-13T14:23:45.933 INFO:tasks.rados.rados.0.smithi026.stdout:1445: delete oid 26 current snap is 132 2024-09-13T14:23:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1445: done (1 left) 2024-09-13T14:23:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 132 2024-09-13T14:23:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1446: seq_num 485 ranges {4096000=532480,4628480=401408} 2024-09-13T14:23:45.971 INFO:tasks.rados.rados.0.smithi026.stdout:1446: writing smithi02683976-18 from 4096000 to 4628480 tid 1 2024-09-13T14:23:45.973 INFO:tasks.rados.rados.0.smithi026.stdout:1446: writing smithi02683976-18 from 4628480 to 5029888 tid 2 2024-09-13T14:23:45.973 INFO:tasks.rados.rados.0.smithi026.stdout:1447: delete oid 1 current snap is 132 2024-09-13T14:23:45.977 INFO:tasks.rados.rados.0.smithi026.stdout:1446: finishing write tid 1 to smithi02683976-18 2024-09-13T14:23:45.977 INFO:tasks.rados.rados.0.smithi026.stdout:1446: finishing write tid 2 to smithi02683976-18 2024-09-13T14:23:45.977 INFO:tasks.rados.rados.0.smithi026.stdout:1446: finishing write tid 3 to smithi02683976-18 2024-09-13T14:23:45.978 INFO:tasks.rados.rados.0.smithi026.stdout:1446: finishing write tid 4 to smithi02683976-18 2024-09-13T14:23:45.978 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 668 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:23:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:1446: left oid 18 (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:23:46.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[99461]: pgmap v95: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:46.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[99461]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:23:46.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[99461]: osd.0 marked itself down and dead 2024-09-13T14:23:46.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[97022]: pgmap v95: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:46.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[97022]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:23:46.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:45 smithi026 ceph-mon[97022]: osd.0 marked itself down and dead 2024-09-13T14:23:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:45 smithi145 ceph-mon[89145]: pgmap v95: 121 pgs: 121 active+clean; 155 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.2 MiB/s wr, 24 op/s 2024-09-13T14:23:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:45 smithi145 ceph-mon[89145]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:23:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:45 smithi145 ceph-mon[89145]: osd.0 marked itself down and dead 2024-09-13T14:23:46.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:46.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:46.887 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:46 smithi026 podman[104662]: 2024-09-13 14:23:46.474687737 +0000 UTC m=+1.421514617 container died db407fd1f6f4dae8b86f1f17e605b834dc067109703370ab234097da8e3083bf (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:23:46.953 INFO:tasks.rados.rados.0.smithi026.stdout:1446: done (2 left) 2024-09-13T14:23:46.953 INFO:tasks.rados.rados.0.smithi026.stdout:1447: done (1 left) 2024-09-13T14:23:46.953 INFO:tasks.rados.rados.0.smithi026.stdout:1448: copy_from oid 39 from oid 1 current snap is 132 2024-09-13T14:23:46.953 INFO:tasks.rados.rados.0.smithi026.stdout:1449: snap_create 2024-09-13T14:23:46.954 INFO:tasks.rados.rados.0.smithi026.stdout:1448: finishing copy_from to smithi02683976-39 2024-09-13T14:23:46.954 INFO:tasks.rados.rados.0.smithi026.stdout:1448: got expected ENOENT (src dne) 2024-09-13T14:23:46.954 INFO:tasks.rados.rados.0.smithi026.stdout:1448: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:23:46.955 INFO:tasks.rados.rados.0.smithi026.stdout:1444: finishing write tid 1 to smithi02683976-45 2024-09-13T14:23:46.955 INFO:tasks.rados.rados.0.smithi026.stdout:1444: finishing write tid 2 to smithi02683976-45 2024-09-13T14:23:46.955 INFO:tasks.rados.rados.0.smithi026.stdout:1444: finishing write tid 3 to smithi02683976-45 2024-09-13T14:23:46.957 INFO:tasks.rados.rados.0.smithi026.stdout:1444: finishing write tid 4 to smithi02683976-45 2024-09-13T14:23:46.957 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 290 (ObjNum 483 snap 132 seq_num 483) dirty exists 2024-09-13T14:23:46.957 INFO:tasks.rados.rados.0.smithi026.stdout:1444: left oid 45 (ObjNum 483 snap 132 seq_num 483) 2024-09-13T14:23:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:46 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:47.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:46 smithi026 ceph-mon[97022]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:23:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:46 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:47.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:46 smithi026 ceph-mon[99461]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:23:47.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:46 smithi026 podman[104662]: 2024-09-13 14:23:46.9936294 +0000 UTC m=+1.940456279 container cleanup db407fd1f6f4dae8b86f1f17e605b834dc067109703370ab234097da8e3083bf (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:47.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:46 smithi026 bash[104662]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:23:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:46 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:46 smithi145 ceph-mon[89145]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:23:47.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:47.567 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104691]: 2024-09-13 14:23:47.253707283 +0000 UTC m=+0.771966021 container remove db407fd1f6f4dae8b86f1f17e605b834dc067109703370ab234097da8e3083bf (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:23:47.890 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104784]: 2024-09-13 14:23:47.470042456 +0000 UTC m=+0.016698012 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:47.890 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104784]: 2024-09-13 14:23:47.587788983 +0000 UTC m=+0.134444577 container create b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T14:23:47.890 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104784]: 2024-09-13 14:23:47.798665469 +0000 UTC m=+0.345321017 container init b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T14:23:47.890 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104784]: 2024-09-13 14:23:47.803611601 +0000 UTC m=+0.350267142 container start b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:23:47.891 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:47 smithi026 podman[104784]: 2024-09-13 14:23:47.890030884 +0000 UTC m=+0.436686423 container attach b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True) 2024-09-13T14:23:47.941 INFO:tasks.rados.rados.0.smithi026.stdout:1444: done (2 left) 2024-09-13T14:23:47.941 INFO:tasks.rados.rados.0.smithi026.stdout:1448: done (1 left) 2024-09-13T14:23:47.942 INFO:tasks.rados.rados.0.smithi026.stdout:1449: done (0 left) 2024-09-13T14:23:47.942 INFO:tasks.rados.rados.0.smithi026.stdout:1450: setattr oid 25 current snap is 133 2024-09-13T14:23:47.943 INFO:tasks.rados.rados.0.smithi026.stdout:1451: setattr oid 47 current snap is 133 2024-09-13T14:23:47.945 INFO:tasks.rados.rados.0.smithi026.stdout:1452: snap_create 2024-09-13T14:23:47.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 598 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:23:47.949 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 302 (ObjNum 476 snap 126 seq_num 476) dirty exists 2024-09-13T14:23:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[97022]: pgmap v98: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[97022]: Health check failed: Reduced data availability: 10 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-13T14:23:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[97022]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T14:23:48.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 podman[104784]: 2024-09-13 14:23:48.003885358 +0000 UTC m=+0.550540896 container died b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T14:23:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[99461]: pgmap v98: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[99461]: Health check failed: Reduced data availability: 10 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-13T14:23:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:47 smithi026 ceph-mon[99461]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T14:23:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:47 smithi145 ceph-mon[89145]: pgmap v98: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:47 smithi145 ceph-mon[89145]: Health check failed: Reduced data availability: 10 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-13T14:23:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:47 smithi145 ceph-mon[89145]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T14:23:48.899 INFO:tasks.rados.rados.0.smithi026.stdout:1450: done (2 left) 2024-09-13T14:23:48.899 INFO:tasks.rados.rados.0.smithi026.stdout:1451: done (1 left) 2024-09-13T14:23:48.899 INFO:tasks.rados.rados.0.smithi026.stdout:1452: done (0 left) 2024-09-13T14:23:48.899 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 134 2024-09-13T14:23:48.899 INFO:tasks.rados.rados.0.smithi026.stdout:1453: seq_num 487 ranges {2842624=565248,3407872=401408} 2024-09-13T14:23:48.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 podman[104827]: 2024-09-13 14:23:48.627585445 +0000 UTC m=+0.618045804 container remove b118dbad7ce881677ca1a4944831c77c4c3278efb91740003c570f8e6d1fa910 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True) 2024-09-13T14:23:48.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:23:48.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 systemd[1]: Stopped Ceph osd.0 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:48.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.0.service: Consumed 12.387s CPU time. 2024-09-13T14:23:48.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:48 smithi026 systemd[1]: Starting Ceph osd.0 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:48.918 INFO:tasks.rados.rados.0.smithi026.stdout:1453: writing smithi02683976-7 from 2842624 to 3407872 tid 1 2024-09-13T14:23:48.921 INFO:tasks.rados.rados.0.smithi026.stdout:1453: writing smithi02683976-7 from 3407872 to 3809280 tid 2 2024-09-13T14:23:48.922 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 134 2024-09-13T14:23:48.922 INFO:tasks.rados.rados.0.smithi026.stdout:1454: seq_num 488 ranges {278528=581632,860160=401408} 2024-09-13T14:23:48.927 INFO:tasks.rados.rados.0.smithi026.stdout:1454: writing smithi02683976-32 from 278528 to 860160 tid 1 2024-09-13T14:23:48.930 INFO:tasks.rados.rados.0.smithi026.stdout:1454: writing smithi02683976-32 from 860160 to 1261568 tid 2 2024-09-13T14:23:48.930 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 134 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:1453: finishing write tid 1 to smithi02683976-7 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:1453: finishing write tid 2 to smithi02683976-7 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:1453: finishing write tid 3 to smithi02683976-7 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:1453: finishing write tid 4 to smithi02683976-7 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 513 (ObjNum 487 snap 134 seq_num 487) dirty exists 2024-09-13T14:23:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:1453: left oid 7 (ObjNum 487 snap 134 seq_num 487) 2024-09-13T14:23:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:1455: seq_num 489 ranges {3801088=598016,4399104=401408} 2024-09-13T14:23:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:1455: writing smithi02683976-34 from 3801088 to 4399104 tid 1 2024-09-13T14:23:48.958 INFO:tasks.rados.rados.0.smithi026.stdout:1455: writing smithi02683976-34 from 4399104 to 4800512 tid 2 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1453: done (2 left) 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1456: snap_create 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1454: finishing write tid 1 to smithi02683976-32 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1454: finishing write tid 2 to smithi02683976-32 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1454: finishing write tid 3 to smithi02683976-32 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1454: finishing write tid 4 to smithi02683976-32 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 672 (ObjNum 488 snap 134 seq_num 488) dirty exists 2024-09-13T14:23:48.959 INFO:tasks.rados.rados.0.smithi026.stdout:1454: left oid 32 (ObjNum 488 snap 134 seq_num 488) 2024-09-13T14:23:48.962 INFO:tasks.rados.rados.0.smithi026.stdout:1455: finishing write tid 1 to smithi02683976-34 2024-09-13T14:23:48.963 INFO:tasks.rados.rados.0.smithi026.stdout:1455: finishing write tid 2 to smithi02683976-34 2024-09-13T14:23:48.963 INFO:tasks.rados.rados.0.smithi026.stdout:1455: finishing write tid 3 to smithi02683976-34 2024-09-13T14:23:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:1455: finishing write tid 4 to smithi02683976-34 2024-09-13T14:23:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 758 (ObjNum 489 snap 134 seq_num 489) dirty exists 2024-09-13T14:23:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:1455: left oid 34 (ObjNum 489 snap 134 seq_num 489) 2024-09-13T14:23:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:48 smithi145 ceph-mon[89145]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T14:23:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:48 smithi145 ceph-mon[89145]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T14:23:49.276 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:48 smithi026 ceph-mon[97022]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T14:23:49.276 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:48 smithi026 ceph-mon[97022]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T14:23:49.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:48 smithi026 ceph-mon[99461]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T14:23:49.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:48 smithi026 ceph-mon[99461]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T14:23:49.277 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.001848257 +0000 UTC m=+0.016336520 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:49.277 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.137464921 +0000 UTC m=+0.151953181 container create 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:23:49.530 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.389913193 +0000 UTC m=+0.404401453 container init 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:23:49.530 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.395123888 +0000 UTC m=+0.409612150 container start 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:23:49.530 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.489960829 +0000 UTC m=+0.504449089 container attach 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:49.917 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:23:49.917 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:23:49.917 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:23:49.917 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:23:49.953 INFO:tasks.rados.rados.0.smithi026.stdout:1454: done (2 left) 2024-09-13T14:23:49.953 INFO:tasks.rados.rados.0.smithi026.stdout:1455: done (1 left) 2024-09-13T14:23:49.953 INFO:tasks.rados.rados.0.smithi026.stdout:1456: done (0 left) 2024-09-13T14:23:49.953 INFO:tasks.rados.rados.0.smithi026.stdout:1457: delete oid 40 current snap is 135 2024-09-13T14:23:49.957 INFO:tasks.rados.rados.0.smithi026.stdout:1457: done (0 left) 2024-09-13T14:23:49.958 INFO:tasks.rados.rados.0.smithi026.stdout:1458: snap_remove snap 130 2024-09-13T14:23:50.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:49 smithi145 ceph-mon[89145]: pgmap v101: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:50.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-mon[99461]: pgmap v101: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:50.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-mon[97022]: pgmap v101: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate[104945]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 bash[104930]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:23:50.245 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:49 smithi026 podman[104930]: 2024-09-13 14:23:49.951514879 +0000 UTC m=+0.966003147 container died 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:23:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:50.855 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 podman[105086]: 2024-09-13 14:23:50.502437005 +0000 UTC m=+0.543104243 container remove 4a06a06cbbc3938d5a23a4be3d2ec9ca7c3b426f061d7b95e2e354579bd75374 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:23:50.855 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 podman[105153]: 2024-09-13 14:23:50.620567055 +0000 UTC m=+0.016837570 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:50.856 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 podman[105153]: 2024-09-13 14:23:50.719857451 +0000 UTC m=+0.116127952 container create 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:23:50.958 INFO:tasks.rados.rados.0.smithi026.stdout:1458: done (0 left) 2024-09-13T14:23:50.959 INFO:tasks.rados.rados.0.smithi026.stdout:1459: snap_remove snap 127 2024-09-13T14:23:51.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:50 smithi026 ceph-mon[99461]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T14:23:51.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:50 smithi026 ceph-mon[97022]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T14:23:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 podman[105153]: 2024-09-13 14:23:50.964156724 +0000 UTC m=+0.360427224 container init 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T14:23:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 podman[105153]: 2024-09-13 14:23:50.96965018 +0000 UTC m=+0.365920680 container start 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T14:23:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:50 smithi026 ceph-osd[105191]: -- 172.21.15.26:0/3480935902 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56211738da40 con 0x56211815a400 2024-09-13T14:23:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:51 smithi026 bash[105153]: 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 2024-09-13T14:23:51.353 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:51 smithi026 systemd[1]: Started Ceph osd.0 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:50 smithi145 ceph-mon[89145]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T14:23:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:23:51] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.51.0" 2024-09-13T14:23:51.959 INFO:tasks.rados.rados.0.smithi026.stdout:1459: done (0 left) 2024-09-13T14:23:51.959 INFO:tasks.rados.rados.0.smithi026.stdout:1460: snap_remove snap 125 2024-09-13T14:23:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[97022]: pgmap v104: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[97022]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[99461]: pgmap v104: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[99461]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:51 smithi145 ceph-mon[89145]: pgmap v104: 121 pgs: 52 peering, 69 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:51 smithi145 ceph-mon[89145]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T14:23:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:52.961 INFO:tasks.rados.rados.0.smithi026.stdout:1460: done (0 left) 2024-09-13T14:23:52.961 INFO:tasks.rados.rados.0.smithi026.stdout:1461: delete oid 34 current snap is 135 2024-09-13T14:23:52.966 INFO:tasks.rados.rados.0.smithi026.stdout:1461: done (0 left) 2024-09-13T14:23:52.967 INFO:tasks.rados.rados.0.smithi026.stdout:1462: read oid 33 snap -1 2024-09-13T14:23:52.967 INFO:tasks.rados.rados.0.smithi026.stdout:1462: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:23:52.967 INFO:tasks.rados.rados.0.smithi026.stdout:1463: snap_remove snap 131 2024-09-13T14:23:53.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:52 smithi026 ceph-mon[97022]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T14:23:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:52 smithi026 ceph-mon[99461]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T14:23:53.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:52 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:23:52.855+0000 7f9edd964640 -1 Falling back to public interface 2024-09-13T14:23:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:52 smithi145 ceph-mon[89145]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T14:23:53.968 INFO:tasks.rados.rados.0.smithi026.stdout:1463: done (1 left) 2024-09-13T14:23:53.968 INFO:tasks.rados.rados.0.smithi026.stdout:1464: setattr oid 2 current snap is 135 2024-09-13T14:23:53.968 INFO:tasks.rados.rados.0.smithi026.stdout:1462: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:23:53.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[99461]: pgmap v107: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:53.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[99461]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T14:23:53.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 44/267 objects degraded (16.479%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:53.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[99461]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T14:23:53.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[97022]: pgmap v107: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:53.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[97022]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T14:23:53.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 44/267 objects degraded (16.479%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:53.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:53 smithi026 ceph-mon[97022]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T14:23:53.996 INFO:tasks.rados.rados.0.smithi026.stdout:1462: done (1 left) 2024-09-13T14:23:53.996 INFO:tasks.rados.rados.0.smithi026.stdout:1465: copy_from oid 12 from oid 18 current snap is 135 2024-09-13T14:23:53.996 INFO:tasks.rados.rados.0.smithi026.stdout:1466: read oid 25 snap -1 2024-09-13T14:23:53.996 INFO:tasks.rados.rados.0.smithi026.stdout:1466: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:23:53.996 INFO:tasks.rados.rados.0.smithi026.stdout:1467: snap_create 2024-09-13T14:23:53.999 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 599 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:23:54.019 INFO:tasks.rados.rados.0.smithi026.stdout:1466: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:23:54.213 INFO:tasks.rados.rados.0.smithi026.stdout:1465: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:23:54.214 INFO:tasks.rados.rados.0.smithi026.stdout:1465: finishing copy_from to smithi02683976-12 2024-09-13T14:23:54.214 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 668 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:23:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:53 smithi145 ceph-mon[89145]: pgmap v107: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:53 smithi145 ceph-mon[89145]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T14:23:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:53 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 44/267 objects degraded (16.479%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:53 smithi145 ceph-mon[89145]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 12 pgs peering) 2024-09-13T14:23:54.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:54.977 INFO:tasks.rados.rados.0.smithi026.stdout:1464: done (3 left) 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1465: done (2 left) 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1466: done (1 left) 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1467: done (0 left) 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1468: rmattr oid 40 current snap is 136 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1468: done (0 left) 2024-09-13T14:23:54.978 INFO:tasks.rados.rados.0.smithi026.stdout:1469: snap_create 2024-09-13T14:23:55.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:54 smithi026 ceph-mon[99461]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T14:23:55.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:54 smithi026 ceph-mon[97022]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T14:23:55.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:54 smithi145 ceph-mon[89145]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T14:23:55.979 INFO:tasks.rados.rados.0.smithi026.stdout:1469: done (0 left) 2024-09-13T14:23:55.979 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 137 2024-09-13T14:23:55.979 INFO:tasks.rados.rados.0.smithi026.stdout:1470: seq_num 492 ranges {8519680=647168,9166848=401408} 2024-09-13T14:23:56.030 INFO:tasks.rados.rados.0.smithi026.stdout:1470: writing smithi02683976-35 from 8519680 to 9166848 tid 1 2024-09-13T14:23:56.032 INFO:tasks.rados.rados.0.smithi026.stdout:1470: writing smithi02683976-35 from 9166848 to 9568256 tid 2 2024-09-13T14:23:56.033 INFO:tasks.rados.rados.0.smithi026.stdout:1471: delete oid 36 current snap is 137 2024-09-13T14:23:56.035 INFO:tasks.rados.rados.0.smithi026.stdout:1470: finishing write tid 1 to smithi02683976-35 2024-09-13T14:23:56.035 INFO:tasks.rados.rados.0.smithi026.stdout:1470: finishing write tid 2 to smithi02683976-35 2024-09-13T14:23:56.036 INFO:tasks.rados.rados.0.smithi026.stdout:1470: finishing write tid 3 to smithi02683976-35 2024-09-13T14:23:56.038 INFO:tasks.rados.rados.0.smithi026.stdout:1470: finishing write tid 4 to smithi02683976-35 2024-09-13T14:23:56.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 823 (ObjNum 492 snap 137 seq_num 492) dirty exists 2024-09-13T14:23:56.038 INFO:tasks.rados.rados.0.smithi026.stdout:1470: left oid 35 (ObjNum 492 snap 137 seq_num 492) 2024-09-13T14:23:56.039 INFO:tasks.rados.rados.0.smithi026.stdout:1470: done (1 left) 2024-09-13T14:23:56.039 INFO:tasks.rados.rados.0.smithi026.stdout:1471: done (0 left) 2024-09-13T14:23:56.039 INFO:tasks.rados.rados.0.smithi026.stdout:1472: read oid 47 snap -1 2024-09-13T14:23:56.039 INFO:tasks.rados.rados.0.smithi026.stdout:1472: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:23:56.039 INFO:tasks.rados.rados.0.smithi026.stdout:1473: snap_remove snap 134 2024-09-13T14:23:56.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: pgmap v110: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:56.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T14:23:56.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.210 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.210 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:56.210 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: pgmap v110: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: pgmap v110: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 938 KiB/s wr, 14 op/s; 44/267 objects degraded (16.479%) 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:56.665 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:23:56.557+0000 7f9edd964640 -1 osd.0 332 log_to_monitors true 2024-09-13T14:23:56.993 INFO:tasks.rados.rados.0.smithi026.stdout:1473: done (1 left) 2024-09-13T14:23:56.993 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 137 2024-09-13T14:23:56.993 INFO:tasks.rados.rados.0.smithi026.stdout:1474: seq_num 494 ranges {0=679936,679936=401408} 2024-09-13T14:23:56.998 INFO:tasks.rados.rados.0.smithi026.stdout:1474: writing smithi02683976-13 from 0 to 679936 tid 1 2024-09-13T14:23:57.001 INFO:tasks.rados.rados.0.smithi026.stdout:1474: writing smithi02683976-13 from 679936 to 1081344 tid 2 2024-09-13T14:23:57.001 INFO:tasks.rados.rados.0.smithi026.stdout:1475: rmattr oid 6 current snap is 137 2024-09-13T14:23:57.001 INFO:tasks.rados.rados.0.smithi026.stdout:1475: done (2 left) 2024-09-13T14:23:57.001 INFO:tasks.rados.rados.0.smithi026.stdout:1476: read oid 33 snap 132 2024-09-13T14:23:57.002 INFO:tasks.rados.rados.0.smithi026.stdout:1476: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:23:57.002 INFO:tasks.rados.rados.0.smithi026.stdout:1477: snap_remove snap 133 2024-09-13T14:23:57.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:23:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-mon[89145]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T14:23:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-mon[89145]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:56 smithi145 ceph-mon[89145]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[99461]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T14:23:57.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:57.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[99461]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[99461]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[97022]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[97022]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:56 smithi026 ceph-mon[97022]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:23:57.547 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:23:57.285+0000 7f9ed1706640 -1 osd.0 332 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:23:58.004 INFO:tasks.rados.rados.0.smithi026.stdout:1477: done (3 left) 2024-09-13T14:23:58.004 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 137 2024-09-13T14:23:58.004 INFO:tasks.rados.rados.0.smithi026.stdout:1478: seq_num 495 ranges {1474560=696320,2170880=401408} 2024-09-13T14:23:58.017 INFO:tasks.rados.rados.0.smithi026.stdout:1478: writing smithi02683976-46 from 1474560 to 2170880 tid 1 2024-09-13T14:23:58.020 INFO:tasks.rados.rados.0.smithi026.stdout:1478: writing smithi02683976-46 from 2170880 to 2572288 tid 2 2024-09-13T14:23:58.021 INFO:tasks.rados.rados.0.smithi026.stdout:1472: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:23:58.171 INFO:tasks.rados.rados.0.smithi026.stdout:1474: finishing write tid 1 to smithi02683976-13 2024-09-13T14:23:58.171 INFO:tasks.rados.rados.0.smithi026.stdout:1474: finishing write tid 2 to smithi02683976-13 2024-09-13T14:23:58.171 INFO:tasks.rados.rados.0.smithi026.stdout:1474: finishing write tid 3 to smithi02683976-13 2024-09-13T14:23:58.171 INFO:tasks.rados.rados.0.smithi026.stdout:1476: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:1474: finishing write tid 4 to smithi02683976-13 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 709 (ObjNum 494 snap 137 seq_num 494) dirty exists 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:1474: left oid 13 (ObjNum 494 snap 137 seq_num 494) 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:1472: done (3 left) 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:1474: done (2 left) 2024-09-13T14:23:58.196 INFO:tasks.rados.rados.0.smithi026.stdout:1476: done (1 left) 2024-09-13T14:23:58.197 INFO:tasks.rados.rados.0.smithi026.stdout:1479: read oid 43 snap -1 2024-09-13T14:23:58.197 INFO:tasks.rados.rados.0.smithi026.stdout:1479: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:23:58.197 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 137 2024-09-13T14:23:58.197 INFO:tasks.rados.rados.0.smithi026.stdout:1480: seq_num 496 ranges {0=712704,712704=401408} 2024-09-13T14:23:58.199 INFO:tasks.rados.rados.0.smithi026.stdout:1480: writing smithi02683976-9 from 0 to 712704 tid 1 2024-09-13T14:23:58.202 INFO:tasks.rados.rados.0.smithi026.stdout:1480: writing smithi02683976-9 from 712704 to 1114112 tid 2 2024-09-13T14:23:58.202 INFO:tasks.rados.rados.0.smithi026.stdout:1481: read oid 41 snap -1 2024-09-13T14:23:58.202 INFO:tasks.rados.rados.0.smithi026.stdout:1481: expect deleted 2024-09-13T14:23:58.202 INFO:tasks.rados.rados.0.smithi026.stdout:1482: snap_create 2024-09-13T14:23:58.205 INFO:tasks.rados.rados.0.smithi026.stdout:1480: finishing write tid 1 to smithi02683976-9 2024-09-13T14:23:58.205 INFO:tasks.rados.rados.0.smithi026.stdout:1480: finishing write tid 2 to smithi02683976-9 2024-09-13T14:23:58.205 INFO:tasks.rados.rados.0.smithi026.stdout:1480: finishing write tid 3 to smithi02683976-9 2024-09-13T14:23:58.207 INFO:tasks.rados.rados.0.smithi026.stdout:1480: finishing write tid 4 to smithi02683976-9 2024-09-13T14:23:58.207 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 775 (ObjNum 496 snap 137 seq_num 496) dirty exists 2024-09-13T14:23:58.207 INFO:tasks.rados.rados.0.smithi026.stdout:1480: left oid 9 (ObjNum 496 snap 137 seq_num 496) 2024-09-13T14:23:58.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: pgmap v113: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:23:58.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[99461]: from='osd.0 ' entity='osd.0' 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: pgmap v113: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:57 smithi026 ceph-mon[97022]: from='osd.0 ' entity='osd.0' 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: pgmap v113: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: from='osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:23:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:57 smithi145 ceph-mon[89145]: from='osd.0 ' entity='osd.0' 2024-09-13T14:23:59.014 INFO:tasks.rados.rados.0.smithi026.stdout:1480: done (4 left) 2024-09-13T14:23:59.014 INFO:tasks.rados.rados.0.smithi026.stdout:1481: done (3 left) 2024-09-13T14:23:59.015 INFO:tasks.rados.rados.0.smithi026.stdout:1482: done (2 left) 2024-09-13T14:23:59.015 INFO:tasks.rados.rados.0.smithi026.stdout:1483: snap_remove snap 132 2024-09-13T14:23:59.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:23:59.146+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775] boot 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775] boot 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: osd.0 [v2:172.21.15.26:6802/3754976775,v1:172.21.15.26:6803/3754976775] boot 2024-09-13T14:23:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:1483: done (2 left) 2024-09-13T14:24:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:1484: rollback oid 12 current snap is 138 2024-09-13T14:24:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 136 2024-09-13T14:24:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:1485: snap_remove snap 135 2024-09-13T14:24:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: pgmap v116: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:24:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:24:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: pgmap v116: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:00.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:00.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:23:59 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:24:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: pgmap v116: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 10 op/s; 47/270 objects degraded (17.407%) 2024-09-13T14:24:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:23:59 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:24:01.017 INFO:tasks.rados.rados.0.smithi026.stdout:1485: done (3 left) 2024-09-13T14:24:01.017 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 138 2024-09-13T14:24:01.017 INFO:tasks.rados.rados.0.smithi026.stdout:1486: seq_num 497 ranges {0=729088,729088=401408} 2024-09-13T14:24:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:1486: writing smithi02683976-24 from 0 to 729088 tid 1 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1486: writing smithi02683976-24 from 729088 to 1130496 tid 2 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1478: finishing write tid 1 to smithi02683976-46 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1478: finishing write tid 2 to smithi02683976-46 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1478: finishing write tid 3 to smithi02683976-46 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1478: finishing write tid 4 to smithi02683976-46 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 443 (ObjNum 495 snap 137 seq_num 495) dirty exists 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1478: left oid 46 (ObjNum 495 snap 137 seq_num 495) 2024-09-13T14:24:01.024 INFO:tasks.rados.rados.0.smithi026.stdout:1479: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:24:01.173 INFO:tasks.rados.rados.0.smithi026.stdout:1484: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:24:01.173 INFO:tasks.rados.rados.0.smithi026.stdout:1484: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1484: finishing rollback tid 2 to smithi02683976-12 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 759 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1478: done (3 left) 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1479: done (2 left) 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1484: done (1 left) 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1487: copy_from oid 10 from oid 5 current snap is 138 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: finishing write tid 1 to smithi02683976-24 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: finishing write tid 2 to smithi02683976-24 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: finishing write tid 3 to smithi02683976-24 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: finishing write tid 4 to smithi02683976-24 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 602 (ObjNum 497 snap 138 seq_num 497) dirty exists 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: left oid 24 (ObjNum 497 snap 138 seq_num 497) 2024-09-13T14:24:01.174 INFO:tasks.rados.rados.0.smithi026.stdout:1486: done (1 left) 2024-09-13T14:24:01.175 INFO:tasks.rados.rados.0.smithi026.stdout:1488: rollback oid 43 current snap is 138 2024-09-13T14:24:01.175 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 43 to 136 2024-09-13T14:24:01.175 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 138 2024-09-13T14:24:01.175 INFO:tasks.rados.rados.0.smithi026.stdout:1489: seq_num 498 ranges {1654784=745472,2400256=401408} 2024-09-13T14:24:01.187 INFO:tasks.rados.rados.0.smithi026.stdout:1489: writing smithi02683976-8 from 1654784 to 2400256 tid 1 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1489: writing smithi02683976-8 from 2400256 to 2801664 tid 2 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 138 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1488: finishing rollback tid 0 to smithi02683976-43 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1488: finishing rollback tid 1 to smithi02683976-43 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1488: finishing rollback tid 2 to smithi02683976-43 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 713 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1487: finishing copy_from racing read to smithi02683976-10 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1487: finishing copy_from to smithi02683976-10 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 796 (ObjNum 467 snap 125 seq_num 467) dirty exists 2024-09-13T14:24:01.190 INFO:tasks.rados.rados.0.smithi026.stdout:1490: seq_num 499 ranges {1114112=761856,1875968=401408} 2024-09-13T14:24:01.200 INFO:tasks.rados.rados.0.smithi026.stdout:1490: writing smithi02683976-9 from 1114112 to 1875968 tid 1 2024-09-13T14:24:01.202 INFO:tasks.rados.rados.0.smithi026.stdout:1490: writing smithi02683976-9 from 1875968 to 2277376 tid 2 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1487: done (3 left) 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1488: done (2 left) 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1491: setattr oid 37 current snap is 138 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1489: finishing write tid 1 to smithi02683976-8 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1489: finishing write tid 2 to smithi02683976-8 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1489: finishing write tid 3 to smithi02683976-8 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1489: finishing write tid 4 to smithi02683976-8 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 582 (ObjNum 498 snap 138 seq_num 498) dirty exists 2024-09-13T14:24:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:1489: left oid 8 (ObjNum 498 snap 138 seq_num 498) 2024-09-13T14:24:01.204 INFO:tasks.rados.rados.0.smithi026.stdout:1489: done (2 left) 2024-09-13T14:24:01.204 INFO:tasks.rados.rados.0.smithi026.stdout:1492: snap_remove snap 137 2024-09-13T14:24:01.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:01 smithi026 ceph-mon[99461]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:24:01.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:01 smithi026 ceph-mon[97022]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:24:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:01 smithi145 ceph-mon[89145]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:24:01.506 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:24:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:01] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.51.0" 2024-09-13T14:24:02.017 INFO:tasks.rados.rados.0.smithi026.stdout:1492: done (2 left) 2024-09-13T14:24:02.017 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 138 2024-09-13T14:24:02.018 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 361 (ObjNum 441 snap 122 seq_num 441) dirty exists 2024-09-13T14:24:02.018 INFO:tasks.rados.rados.0.smithi026.stdout:1493: seq_num 500 ranges {860160=778240,1638400=401408} 2024-09-13T14:24:02.028 INFO:tasks.rados.rados.0.smithi026.stdout:1493: writing smithi02683976-38 from 860160 to 1638400 tid 1 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1493: writing smithi02683976-38 from 1638400 to 2039808 tid 2 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1491: done (2 left) 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1494: setattr oid 24 current snap is 138 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1490: finishing write tid 1 to smithi02683976-9 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1490: finishing write tid 2 to smithi02683976-9 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1490: finishing write tid 3 to smithi02683976-9 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1490: finishing write tid 4 to smithi02683976-9 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 779 (ObjNum 499 snap 138 seq_num 499) dirty exists 2024-09-13T14:24:02.031 INFO:tasks.rados.rados.0.smithi026.stdout:1490: left oid 9 (ObjNum 499 snap 138 seq_num 499) 2024-09-13T14:24:02.033 INFO:tasks.rados.rados.0.smithi026.stdout:1490: done (2 left) 2024-09-13T14:24:02.033 INFO:tasks.rados.rados.0.smithi026.stdout:1495: snap_remove snap 129 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 7s ago 7m 19.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (2m) 48s ago 7m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (8m) 7s ago 8m 15.0M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (8m) 48s ago 8m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (102s) 48s ago 12m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:24:02.094 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (2m) 7s ago 14m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (88s) 7s ago 14m 52.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (53s) 48s ago 12m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (66s) 7s ago 12m 38.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 7s ago 7m 12.0M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (2m) 48s ago 6m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (11s) 7s ago 12m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (11m) 7s ago 11m 166M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8d2e66aacf4c 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (11m) 7s ago 11m 172M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (10m) 7s ago 10m 103M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (10m) 48s ago 10m 171M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (10m) 48s ago 10m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (9m) 48s ago 9m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:24:02.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (9m) 48s ago 9m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:24:02.096 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (107s) 48s ago 7m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:24:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:02 smithi026 ceph-mon[97022]: pgmap v119: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47/270 objects degraded (17.407%) 2024-09-13T14:24:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:02 smithi026 ceph-mon[97022]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:24:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:02 smithi026 ceph-mon[99461]: pgmap v119: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47/270 objects degraded (17.407%) 2024-09-13T14:24:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:02 smithi026 ceph-mon[99461]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:24:02.441 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 7, 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:24:02.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 9, 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 6 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:24:02.443 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:24:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:02 smithi145 ceph-mon[89145]: pgmap v119: 121 pgs: 36 active+undersized, 16 active+undersized+degraded, 69 active+clean; 170 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47/270 objects degraded (17.407%) 2024-09-13T14:24:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:02 smithi145 ceph-mon[89145]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:24:02.738 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "6/20 daemons upgraded", 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:24:02.739 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:24:03.030 INFO:tasks.rados.rados.0.smithi026.stdout:1495: done (2 left) 2024-09-13T14:24:03.030 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 138 2024-09-13T14:24:03.030 INFO:tasks.rados.rados.0.smithi026.stdout:1496: seq_num 501 ranges {3801088=794624,4595712=401408} 2024-09-13T14:24:03.055 INFO:tasks.rados.rados.0.smithi026.stdout:1496: writing smithi02683976-50 from 3801088 to 4595712 tid 1 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1496: writing smithi02683976-50 from 4595712 to 4997120 tid 2 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1497: copy_from oid 28 from oid 19 current snap is 138 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1493: finishing write tid 1 to smithi02683976-38 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1498: setattr oid 47 current snap is 138 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1493: finishing write tid 2 to smithi02683976-38 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:1493: finishing write tid 3 to smithi02683976-38 2024-09-13T14:24:03.058 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 603 (ObjNum 497 snap 138 seq_num 497) dirty exists 2024-09-13T14:24:03.059 INFO:tasks.rados.rados.0.smithi026.stdout:1493: finishing write tid 4 to smithi02683976-38 2024-09-13T14:24:03.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 696 (ObjNum 500 snap 138 seq_num 500) dirty exists 2024-09-13T14:24:03.059 INFO:tasks.rados.rados.0.smithi026.stdout:1493: left oid 38 (ObjNum 500 snap 138 seq_num 500) 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:1493: done (4 left) 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:1494: done (3 left) 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:1499: rollback oid 44 current snap is 138 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 136 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:1500: rollback oid 6 current snap is 138 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 136 2024-09-13T14:24:03.061 INFO:tasks.rados.rados.0.smithi026.stdout:1501: delete oid 7 current snap is 138 2024-09-13T14:24:03.065 INFO:tasks.rados.rados.0.smithi026.stdout:1501: done (5 left) 2024-09-13T14:24:03.065 INFO:tasks.rados.rados.0.smithi026.stdout:1502: read oid 33 snap -1 2024-09-13T14:24:03.065 INFO:tasks.rados.rados.0.smithi026.stdout:1502: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:24:03.065 INFO:tasks.rados.rados.0.smithi026.stdout:1503: delete oid 3 current snap is 138 2024-09-13T14:24:03.066 INFO:tasks.rados.rados.0.smithi026.stdout:1496: finishing write tid 1 to smithi02683976-50 2024-09-13T14:24:03.066 INFO:tasks.rados.rados.0.smithi026.stdout:1496: finishing write tid 2 to smithi02683976-50 2024-09-13T14:24:03.066 INFO:tasks.rados.rados.0.smithi026.stdout:1496: finishing write tid 3 to smithi02683976-50 2024-09-13T14:24:03.066 INFO:tasks.rados.rados.0.smithi026.stdout:1500: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:1500: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 305 (ObjNum 476 snap 126 seq_num 476) dirty exists 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:1500: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 789 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:1496: finishing write tid 4 to smithi02683976-50 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 785 (ObjNum 501 snap 138 seq_num 501) dirty exists 2024-09-13T14:24:03.067 INFO:tasks.rados.rados.0.smithi026.stdout:1496: left oid 50 (ObjNum 501 snap 138 seq_num 501) 2024-09-13T14:24:03.070 INFO:tasks.rados.rados.0.smithi026.stdout:1497: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:1499: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:1497: finishing copy_from to smithi02683976-28 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 706 (ObjNum 479 snap 129 seq_num 479) dirty exists 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:1499: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:1499: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:24:03.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 446 (ObjNum 478 snap 129 seq_num 478) dirty exists 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1496: done (6 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1497: done (5 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1498: done (4 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1499: done (3 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1500: done (2 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1503: done (1 left) 2024-09-13T14:24:03.074 INFO:tasks.rados.rados.0.smithi026.stdout:1504: snap_remove snap 136 2024-09-13T14:24:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[99461]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[99461]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[99461]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[99461]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1924863657' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[97022]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[97022]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[97022]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[97022]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:03 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1924863657' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:03 smithi145 ceph-mon[89145]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:03 smithi145 ceph-mon[89145]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:03 smithi145 ceph-mon[89145]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:24:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:03 smithi145 ceph-mon[89145]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:03 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1924863657' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:03.903 INFO:tasks.rados.rados.0.smithi026.stdout:1504: done (1 left) 2024-09-13T14:24:03.903 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 138 2024-09-13T14:24:03.904 INFO:tasks.rados.rados.0.smithi026.stdout:1505: seq_num 504 ranges {0=450560,450560=589824,1040384=212992} 2024-09-13T14:24:03.906 INFO:tasks.rados.rados.0.smithi026.stdout:1505: writing smithi02683976-3 from 0 to 450560 tid 1 2024-09-13T14:24:03.911 INFO:tasks.rados.rados.0.smithi026.stdout:1505: writing smithi02683976-3 from 450560 to 1040384 tid 2 2024-09-13T14:24:03.912 INFO:tasks.rados.rados.0.smithi026.stdout:1505: writing smithi02683976-3 from 1040384 to 1253376 tid 3 2024-09-13T14:24:03.912 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 138 2024-09-13T14:24:03.912 INFO:tasks.rados.rados.0.smithi026.stdout:1506: seq_num 505 ranges {0=466944,466944=475136,942080=327680} 2024-09-13T14:24:03.915 INFO:tasks.rados.rados.0.smithi026.stdout:1506: writing smithi02683976-20 from 0 to 466944 tid 1 2024-09-13T14:24:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1506: writing smithi02683976-20 from 466944 to 942080 tid 2 2024-09-13T14:24:03.921 INFO:tasks.rados.rados.0.smithi026.stdout:1506: writing smithi02683976-20 from 942080 to 1269760 tid 3 2024-09-13T14:24:03.921 INFO:tasks.rados.rados.0.smithi026.stdout:1507: rollback oid 45 current snap is 138 2024-09-13T14:24:03.921 INFO:tasks.rados.rados.0.smithi026.stdout:1507: done (3 left) 2024-09-13T14:24:03.922 INFO:tasks.rados.rados.0.smithi026.stdout:1508: read oid 37 snap -1 2024-09-13T14:24:03.922 INFO:tasks.rados.rados.0.smithi026.stdout:1508: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:03.922 INFO:tasks.rados.rados.0.smithi026.stdout:1509: snap_create 2024-09-13T14:24:03.924 INFO:tasks.rados.rados.0.smithi026.stdout:1502: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:24:03.953 INFO:tasks.rados.rados.0.smithi026.stdout:1505: finishing write tid 1 to smithi02683976-3 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: finishing write tid 1 to smithi02683976-20 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1505: finishing write tid 2 to smithi02683976-3 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1505: finishing write tid 3 to smithi02683976-3 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1505: finishing write tid 4 to smithi02683976-3 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: finishing write tid 2 to smithi02683976-20 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: finishing write tid 3 to smithi02683976-20 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: finishing write tid 4 to smithi02683976-20 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: finishing write tid 5 to smithi02683976-20 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 723 (ObjNum 505 snap 138 seq_num 505) dirty exists 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1506: left oid 20 (ObjNum 505 snap 138 seq_num 505) 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:1505: finishing write tid 5 to smithi02683976-3 2024-09-13T14:24:03.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 861 (ObjNum 504 snap 138 seq_num 504) dirty exists 2024-09-13T14:24:03.955 INFO:tasks.rados.rados.0.smithi026.stdout:1505: left oid 3 (ObjNum 504 snap 138 seq_num 504) 2024-09-13T14:24:03.955 INFO:tasks.rados.rados.0.smithi026.stdout:1508: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:04.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: pgmap v122: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:04.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded) 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[99461]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: pgmap v122: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded) 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:24:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:04 smithi026 ceph-mon[97022]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:24:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: pgmap v122: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/270 objects degraded (17.407%), 16 pgs degraded) 2024-09-13T14:24:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:24:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:24:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-mon[89145]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:24:04.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:04.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:24:04.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:04.904 INFO:tasks.rados.rados.0.smithi026.stdout:1502: done (4 left) 2024-09-13T14:24:04.904 INFO:tasks.rados.rados.0.smithi026.stdout:1505: done (3 left) 2024-09-13T14:24:04.905 INFO:tasks.rados.rados.0.smithi026.stdout:1506: done (2 left) 2024-09-13T14:24:04.905 INFO:tasks.rados.rados.0.smithi026.stdout:1508: done (1 left) 2024-09-13T14:24:04.905 INFO:tasks.rados.rados.0.smithi026.stdout:1509: done (0 left) 2024-09-13T14:24:04.905 INFO:tasks.rados.rados.0.smithi026.stdout:1510: rollback oid 29 current snap is 139 2024-09-13T14:24:04.906 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 138 2024-09-13T14:24:04.906 INFO:tasks.rados.rados.0.smithi026.stdout:1511: read oid 47 snap -1 2024-09-13T14:24:04.906 INFO:tasks.rados.rados.0.smithi026.stdout:1511: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:24:04.906 INFO:tasks.rados.rados.0.smithi026.stdout:1512: delete oid 13 current snap is 139 2024-09-13T14:24:04.910 INFO:tasks.rados.rados.0.smithi026.stdout:1510: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:24:04.910 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 0 (ObjNum 53697 snap 0 seq_num 3890193488) dirty dne 2024-09-13T14:24:04.911 INFO:tasks.rados.rados.0.smithi026.stdout:1510: done (2 left) 2024-09-13T14:24:04.911 INFO:tasks.rados.rados.0.smithi026.stdout:1512: done (1 left) 2024-09-13T14:24:04.911 INFO:tasks.rados.rados.0.smithi026.stdout:1513: delete oid 22 current snap is 139 2024-09-13T14:24:04.915 INFO:tasks.rados.rados.0.smithi026.stdout:1513: done (1 left) 2024-09-13T14:24:04.916 INFO:tasks.rados.rados.0.smithi026.stdout:1514: read oid 8 snap -1 2024-09-13T14:24:04.916 INFO:tasks.rados.rados.0.smithi026.stdout:1514: expect (ObjNum 498 snap 138 seq_num 498) 2024-09-13T14:24:04.916 INFO:tasks.rados.rados.0.smithi026.stdout:1515: snap_create 2024-09-13T14:24:04.917 INFO:tasks.rados.rados.0.smithi026.stdout:1511: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:24:05.067 INFO:tasks.rados.rados.0.smithi026.stdout:1514: expect (ObjNum 498 snap 138 seq_num 498) 2024-09-13T14:24:05.912 INFO:tasks.rados.rados.0.smithi026.stdout:1511: done (2 left) 2024-09-13T14:24:05.912 INFO:tasks.rados.rados.0.smithi026.stdout:1514: done (1 left) 2024-09-13T14:24:05.912 INFO:tasks.rados.rados.0.smithi026.stdout:1515: done (0 left) 2024-09-13T14:24:05.912 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 140 2024-09-13T14:24:05.912 INFO:tasks.rados.rados.0.smithi026.stdout:1516: seq_num 508 ranges {6987776=516096,7503872=524288,8028160=278528} 2024-09-13T14:24:05.952 INFO:tasks.rados.rados.0.smithi026.stdout:1516: writing smithi02683976-44 from 6987776 to 7503872 tid 1 2024-09-13T14:24:05.955 INFO:tasks.rados.rados.0.smithi026.stdout:1516: writing smithi02683976-44 from 7503872 to 8028160 tid 2 2024-09-13T14:24:05.957 INFO:tasks.rados.rados.0.smithi026.stdout:1516: writing smithi02683976-44 from 8028160 to 8306688 tid 3 2024-09-13T14:24:05.957 INFO:tasks.rados.rados.0.smithi026.stdout:1517: delete oid 45 current snap is 140 2024-09-13T14:24:05.960 INFO:tasks.rados.rados.0.smithi026.stdout:1517: done (1 left) 2024-09-13T14:24:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:1518: snap_create 2024-09-13T14:24:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:1516: finishing write tid 1 to smithi02683976-44 2024-09-13T14:24:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:1516: finishing write tid 2 to smithi02683976-44 2024-09-13T14:24:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:1516: finishing write tid 3 to smithi02683976-44 2024-09-13T14:24:05.962 INFO:tasks.rados.rados.0.smithi026.stdout:1516: finishing write tid 4 to smithi02683976-44 2024-09-13T14:24:05.963 INFO:tasks.rados.rados.0.smithi026.stdout:1516: finishing write tid 5 to smithi02683976-44 2024-09-13T14:24:05.963 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 450 (ObjNum 508 snap 140 seq_num 508) dirty exists 2024-09-13T14:24:05.963 INFO:tasks.rados.rados.0.smithi026.stdout:1516: left oid 44 (ObjNum 508 snap 140 seq_num 508) 2024-09-13T14:24:06.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:05 smithi026 ceph-mon[97022]: pgmap v125: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:06.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:05 smithi026 ceph-mon[97022]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:24:06.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:05 smithi026 ceph-mon[99461]: pgmap v125: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:06.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:05 smithi026 ceph-mon[99461]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:24:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:05 smithi145 ceph-mon[89145]: pgmap v125: 121 pgs: 121 active+clean; 178 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.2 MiB/s wr, 18 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-13T14:24:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:05 smithi145 ceph-mon[89145]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:24:06.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:06.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:06.916 INFO:tasks.rados.rados.0.smithi026.stdout:1516: done (1 left) 2024-09-13T14:24:06.916 INFO:tasks.rados.rados.0.smithi026.stdout:1518: done (0 left) 2024-09-13T14:24:06.916 INFO:tasks.rados.rados.0.smithi026.stdout:1519: snap_remove snap 138 2024-09-13T14:24:07.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:06 smithi026 ceph-mon[97022]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:24:07.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:06 smithi026 ceph-mon[99461]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:24:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:06 smithi145 ceph-mon[89145]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:24:07.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:07.918 INFO:tasks.rados.rados.0.smithi026.stdout:1519: done (0 left) 2024-09-13T14:24:07.919 INFO:tasks.rados.rados.0.smithi026.stdout:1520: snap_create 2024-09-13T14:24:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:07 smithi145 ceph-mon[89145]: pgmap v128: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:24:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:07 smithi145 ceph-mon[89145]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:24:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:07 smithi026 ceph-mon[97022]: pgmap v128: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:24:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:07 smithi026 ceph-mon[97022]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:24:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:07 smithi026 ceph-mon[99461]: pgmap v128: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:24:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:07 smithi026 ceph-mon[99461]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:24:08.904 INFO:tasks.rados.rados.0.smithi026.stdout:1520: done (0 left) 2024-09-13T14:24:08.905 INFO:tasks.rados.rados.0.smithi026.stdout:1521: read oid 11 snap -1 2024-09-13T14:24:08.905 INFO:tasks.rados.rados.0.smithi026.stdout:1521: expect deleted 2024-09-13T14:24:08.905 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 142 2024-09-13T14:24:08.905 INFO:tasks.rados.rados.0.smithi026.stdout:1522: seq_num 510 ranges {1130496=548864,1679360=745472,2424832=57344} 2024-09-13T14:24:08.913 INFO:tasks.rados.rados.0.smithi026.stdout:1522: writing smithi02683976-24 from 1130496 to 1679360 tid 1 2024-09-13T14:24:08.919 INFO:tasks.rados.rados.0.smithi026.stdout:1522: writing smithi02683976-24 from 1679360 to 2424832 tid 2 2024-09-13T14:24:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:1522: writing smithi02683976-24 from 2424832 to 2482176 tid 3 2024-09-13T14:24:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 142 2024-09-13T14:24:08.920 INFO:tasks.rados.rados.0.smithi026.stdout:1523: seq_num 511 ranges {1261568=565248,1826816=630784,2457600=172032} 2024-09-13T14:24:08.929 INFO:tasks.rados.rados.0.smithi026.stdout:1523: writing smithi02683976-32 from 1261568 to 1826816 tid 1 2024-09-13T14:24:08.933 INFO:tasks.rados.rados.0.smithi026.stdout:1523: writing smithi02683976-32 from 1826816 to 2457600 tid 2 2024-09-13T14:24:08.935 INFO:tasks.rados.rados.0.smithi026.stdout:1523: writing smithi02683976-32 from 2457600 to 2629632 tid 3 2024-09-13T14:24:08.935 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 142 2024-09-13T14:24:08.935 INFO:tasks.rados.rados.0.smithi026.stdout:1524: seq_num 512 ranges {0=581632,581632=516096,1097728=286720} 2024-09-13T14:24:08.938 INFO:tasks.rados.rados.0.smithi026.stdout:1524: writing smithi02683976-36 from 0 to 581632 tid 1 2024-09-13T14:24:08.941 INFO:tasks.rados.rados.0.smithi026.stdout:1524: writing smithi02683976-36 from 581632 to 1097728 tid 2 2024-09-13T14:24:08.943 INFO:tasks.rados.rados.0.smithi026.stdout:1524: writing smithi02683976-36 from 1097728 to 1384448 tid 3 2024-09-13T14:24:08.943 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 142 2024-09-13T14:24:08.943 INFO:tasks.rados.rados.0.smithi026.stdout:1525: seq_num 513 ranges {0=598016,598016=794624,1392640=8192} 2024-09-13T14:24:08.947 INFO:tasks.rados.rados.0.smithi026.stdout:1525: writing smithi02683976-22 from 0 to 598016 tid 1 2024-09-13T14:24:08.952 INFO:tasks.rados.rados.0.smithi026.stdout:1525: writing smithi02683976-22 from 598016 to 1392640 tid 2 2024-09-13T14:24:08.952 INFO:tasks.rados.rados.0.smithi026.stdout:1525: writing smithi02683976-22 from 1392640 to 1400832 tid 3 2024-09-13T14:24:08.952 INFO:tasks.rados.rados.0.smithi026.stdout:1526: snap_remove snap 140 2024-09-13T14:24:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:08 smithi145 ceph-mon[89145]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:24:09.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:08 smithi145 ceph-mon[89145]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:24:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:08 smithi026 ceph-mon[97022]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:24:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:08 smithi026 ceph-mon[97022]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:24:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:08 smithi026 ceph-mon[99461]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:24:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:08 smithi026 ceph-mon[99461]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:24:09.940 INFO:tasks.rados.rados.0.smithi026.stdout:1526: done (5 left) 2024-09-13T14:24:09.940 INFO:tasks.rados.rados.0.smithi026.stdout:1527: rollback oid 34 current snap is 142 2024-09-13T14:24:09.940 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 141 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: finishing write tid 1 to smithi02683976-24 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: finishing write tid 2 to smithi02683976-24 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: finishing write tid 3 to smithi02683976-24 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: finishing write tid 4 to smithi02683976-24 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: finishing write tid 5 to smithi02683976-24 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 607 (ObjNum 510 snap 142 seq_num 510) dirty exists 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: left oid 24 (ObjNum 510 snap 142 seq_num 510) 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1523: finishing write tid 1 to smithi02683976-32 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1521: done (5 left) 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1522: done (4 left) 2024-09-13T14:24:09.941 INFO:tasks.rados.rados.0.smithi026.stdout:1528: snap_remove snap 139 2024-09-13T14:24:10.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:09 smithi145 ceph-mon[89145]: pgmap v131: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-13T14:24:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:09 smithi026 ceph-mon[97022]: pgmap v131: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-13T14:24:10.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:09 smithi026 ceph-mon[99461]: pgmap v131: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-13T14:24:10.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1528: done (4 left) 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1529: read oid 13 snap 141 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1529: expect deleted 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 142 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1523: finishing write tid 2 to smithi02683976-32 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1523: finishing write tid 3 to smithi02683976-32 2024-09-13T14:24:10.945 INFO:tasks.rados.rados.0.smithi026.stdout:1523: finishing write tid 4 to smithi02683976-32 2024-09-13T14:24:10.946 INFO:tasks.rados.rados.0.smithi026.stdout:1530: seq_num 514 ranges {4243456=614400,4857856=679936,5537792=122880} 2024-09-13T14:24:10.971 INFO:tasks.rados.rados.0.smithi026.stdout:1530: writing smithi02683976-6 from 4243456 to 4857856 tid 1 2024-09-13T14:24:10.975 INFO:tasks.rados.rados.0.smithi026.stdout:1530: writing smithi02683976-6 from 4857856 to 5537792 tid 2 2024-09-13T14:24:10.976 INFO:tasks.rados.rados.0.smithi026.stdout:1530: writing smithi02683976-6 from 5537792 to 5660672 tid 3 2024-09-13T14:24:10.976 INFO:tasks.rados.rados.0.smithi026.stdout:1531: copy_from oid 26 from oid 12 current snap is 142 2024-09-13T14:24:10.976 INFO:tasks.rados.rados.0.smithi026.stdout:1523: finishing write tid 5 to smithi02683976-32 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 707 (ObjNum 511 snap 142 seq_num 511) dirty exists 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1523: left oid 32 (ObjNum 511 snap 142 seq_num 511) 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: finishing write tid 1 to smithi02683976-36 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: finishing write tid 2 to smithi02683976-36 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: finishing write tid 3 to smithi02683976-36 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: finishing write tid 4 to smithi02683976-36 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: finishing write tid 5 to smithi02683976-36 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 871 (ObjNum 512 snap 142 seq_num 512) dirty exists 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1524: left oid 36 (ObjNum 512 snap 142 seq_num 512) 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1525: finishing write tid 1 to smithi02683976-22 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1525: finishing write tid 2 to smithi02683976-22 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1525: finishing write tid 3 to smithi02683976-22 2024-09-13T14:24:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:1523: done (6 left) 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1524: done (5 left) 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1532: setattr oid 41 current snap is 142 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1525: finishing write tid 4 to smithi02683976-22 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1525: finishing write tid 5 to smithi02683976-22 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 764 (ObjNum 513 snap 142 seq_num 513) dirty exists 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1525: left oid 22 (ObjNum 513 snap 142 seq_num 513) 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:1527: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:24:10.978 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 0 (ObjNum 385 snap 0 seq_num 519425680) dirty dne 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1525: done (5 left) 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1527: done (4 left) 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1529: done (3 left) 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1533: read oid 8 snap -1 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1533: expect (ObjNum 498 snap 138 seq_num 498) 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1534: read oid 25 snap -1 2024-09-13T14:24:10.979 INFO:tasks.rados.rados.0.smithi026.stdout:1534: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:24:10.980 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 142 2024-09-13T14:24:10.980 INFO:tasks.rados.rados.0.smithi026.stdout:1535: seq_num 515 ranges {2629632=630784,3260416=565248,3825664=237568} 2024-09-13T14:24:10.996 INFO:tasks.rados.rados.0.smithi026.stdout:1535: writing smithi02683976-32 from 2629632 to 3260416 tid 1 2024-09-13T14:24:11.000 INFO:tasks.rados.rados.0.smithi026.stdout:1535: writing smithi02683976-32 from 3260416 to 3825664 tid 2 2024-09-13T14:24:11.002 INFO:tasks.rados.rados.0.smithi026.stdout:1535: writing smithi02683976-32 from 3825664 to 4063232 tid 3 2024-09-13T14:24:11.002 INFO:tasks.rados.rados.0.smithi026.stdout:1536: setattr oid 29 current snap is 142 2024-09-13T14:24:11.002 INFO:tasks.rados.rados.0.smithi026.stdout:1530: finishing write tid 1 to smithi02683976-6 2024-09-13T14:24:11.002 INFO:tasks.rados.rados.0.smithi026.stdout:1530: finishing write tid 2 to smithi02683976-6 2024-09-13T14:24:11.002 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 760 (ObjNum 705 snap 0 seq_num 519609440) dirty exists 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1530: finishing write tid 3 to smithi02683976-6 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1530: finishing write tid 4 to smithi02683976-6 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1530: finishing write tid 5 to smithi02683976-6 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 798 (ObjNum 514 snap 142 seq_num 514) dirty exists 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1530: left oid 6 (ObjNum 514 snap 142 seq_num 514) 2024-09-13T14:24:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1531: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:24:11.004 INFO:tasks.rados.rados.0.smithi026.stdout:1530: done (6 left) 2024-09-13T14:24:11.004 INFO:tasks.rados.rados.0.smithi026.stdout:1532: done (5 left) 2024-09-13T14:24:11.004 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 142 2024-09-13T14:24:11.004 INFO:tasks.rados.rados.0.smithi026.stdout:1537: seq_num 516 ranges {1843200=647168,2490368=450560,2940928=352256} 2024-09-13T14:24:11.018 INFO:tasks.rados.rados.0.smithi026.stdout:1537: writing smithi02683976-31 from 1843200 to 2490368 tid 1 2024-09-13T14:24:11.020 INFO:tasks.rados.rados.0.smithi026.stdout:1537: writing smithi02683976-31 from 2490368 to 2940928 tid 2 2024-09-13T14:24:11.022 INFO:tasks.rados.rados.0.smithi026.stdout:1537: writing smithi02683976-31 from 2940928 to 3293184 tid 3 2024-09-13T14:24:11.022 INFO:tasks.rados.rados.0.smithi026.stdout:1538: copy_from oid 34 from oid 50 current snap is 142 2024-09-13T14:24:11.022 INFO:tasks.rados.rados.0.smithi026.stdout:1533: expect (ObjNum 498 snap 138 seq_num 498) 2024-09-13T14:24:11.161 INFO:tasks.rados.rados.0.smithi026.stdout:1531: finishing copy_from to smithi02683976-26 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 759 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1535: finishing write tid 1 to smithi02683976-32 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1531: done (6 left) 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1533: done (5 left) 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1539: read oid 40 snap 141 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1539: expect deleted 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1540: rmattr oid 43 current snap is 142 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1540: done (6 left) 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 142 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 729 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1535: finishing write tid 2 to smithi02683976-32 2024-09-13T14:24:11.162 INFO:tasks.rados.rados.0.smithi026.stdout:1541: seq_num 517 ranges {827392=663552,1490944=786432,2277376=16384} 2024-09-13T14:24:11.170 INFO:tasks.rados.rados.0.smithi026.stdout:1541: writing smithi02683976-28 from 827392 to 1490944 tid 1 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1541: writing smithi02683976-28 from 1490944 to 2277376 tid 2 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1541: writing smithi02683976-28 from 2277376 to 2293760 tid 3 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1536: done (6 left) 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1542: read oid 35 snap -1 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1542: expect (ObjNum 492 snap 137 seq_num 492) 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 142 2024-09-13T14:24:11.176 INFO:tasks.rados.rados.0.smithi026.stdout:1534: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:24:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:10 smithi145 ceph-mon[89145]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:24:11.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:10 smithi026 ceph-mon[99461]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:24:11.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:10 smithi026 ceph-mon[97022]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:24:11.375 INFO:tasks.rados.rados.0.smithi026.stdout:1535: finishing write tid 3 to smithi02683976-32 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:1535: finishing write tid 4 to smithi02683976-32 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:1535: finishing write tid 5 to smithi02683976-32 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 715 (ObjNum 515 snap 142 seq_num 515) dirty exists 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:1535: left oid 32 (ObjNum 515 snap 142 seq_num 515) 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:1537: finishing write tid 1 to smithi02683976-31 2024-09-13T14:24:11.376 INFO:tasks.rados.rados.0.smithi026.stdout:1543: seq_num 518 ranges {3112960=679936,3792896=671744,4464640=131072} 2024-09-13T14:24:11.396 INFO:tasks.rados.rados.0.smithi026.stdout:1543: writing smithi02683976-16 from 3112960 to 3792896 tid 1 2024-09-13T14:24:11.400 INFO:tasks.rados.rados.0.smithi026.stdout:1543: writing smithi02683976-16 from 3792896 to 4464640 tid 2 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1543: writing smithi02683976-16 from 4464640 to 4595712 tid 3 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1534: done (7 left) 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1535: done (6 left) 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1544: rollback oid 27 current snap is 142 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 141 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1537: finishing write tid 2 to smithi02683976-31 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1537: finishing write tid 3 to smithi02683976-31 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1537: finishing write tid 4 to smithi02683976-31 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:1537: finishing write tid 5 to smithi02683976-31 2024-09-13T14:24:11.402 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 803 (ObjNum 516 snap 142 seq_num 516) dirty exists 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1537: left oid 31 (ObjNum 516 snap 142 seq_num 516) 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1538: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1538: finishing copy_from to smithi02683976-34 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 785 (ObjNum 501 snap 138 seq_num 501) dirty exists 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1537: done (6 left) 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1538: done (5 left) 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1545: rollback oid 34 current snap is 142 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 141 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1541: finishing write tid 1 to smithi02683976-28 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1541: finishing write tid 2 to smithi02683976-28 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1541: finishing write tid 3 to smithi02683976-28 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1541: finishing write tid 4 to smithi02683976-28 2024-09-13T14:24:11.403 INFO:tasks.rados.rados.0.smithi026.stdout:1541: finishing write tid 5 to smithi02683976-28 2024-09-13T14:24:11.404 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 800 (ObjNum 517 snap 142 seq_num 517) dirty exists 2024-09-13T14:24:11.404 INFO:tasks.rados.rados.0.smithi026.stdout:1541: left oid 28 (ObjNum 517 snap 142 seq_num 517) 2024-09-13T14:24:11.404 INFO:tasks.rados.rados.0.smithi026.stdout:1542: expect (ObjNum 492 snap 137 seq_num 492) 2024-09-13T14:24:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:11] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-13T14:24:12.188 INFO:tasks.rados.rados.0.smithi026.stdout:1543: finishing write tid 1 to smithi02683976-16 2024-09-13T14:24:12.188 INFO:tasks.rados.rados.0.smithi026.stdout:1545: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:24:12.188 INFO:tasks.rados.rados.0.smithi026.stdout:1545: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:24:12.188 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 787 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1543: finishing write tid 2 to smithi02683976-16 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1543: finishing write tid 3 to smithi02683976-16 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1539: done (5 left) 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1541: done (4 left) 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1542: done (3 left) 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1545: done (2 left) 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1546: rmattr oid 49 current snap is 142 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1546: done (2 left) 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 142 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1544: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:24:12.189 INFO:tasks.rados.rados.0.smithi026.stdout:1547: seq_num 519 ranges {1269760=696320,1966080=548864,2514944=253952} 2024-09-13T14:24:12.198 INFO:tasks.rados.rados.0.smithi026.stdout:1547: writing smithi02683976-20 from 1269760 to 1966080 tid 1 2024-09-13T14:24:12.202 INFO:tasks.rados.rados.0.smithi026.stdout:1547: writing smithi02683976-20 from 1966080 to 2514944 tid 2 2024-09-13T14:24:12.204 INFO:tasks.rados.rados.0.smithi026.stdout:1547: writing smithi02683976-20 from 2514944 to 2768896 tid 3 2024-09-13T14:24:12.204 INFO:tasks.rados.rados.0.smithi026.stdout:1548: read oid 4 snap -1 2024-09-13T14:24:12.204 INFO:tasks.rados.rados.0.smithi026.stdout:1548: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1549: rollback oid 45 current snap is 142 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 141 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1550: read oid 7 snap -1 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1550: expect deleted 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1551: snap_create 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1543: finishing write tid 4 to smithi02683976-16 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1544: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1544: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 875 (ObjNum 482 snap 131 seq_num 482) dirty exists 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1543: finishing write tid 5 to smithi02683976-16 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 734 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:12.205 INFO:tasks.rados.rados.0.smithi026.stdout:1543: left oid 16 (ObjNum 518 snap 142 seq_num 518) 2024-09-13T14:24:12.206 INFO:tasks.rados.rados.0.smithi026.stdout:1547: finishing write tid 1 to smithi02683976-20 2024-09-13T14:24:12.206 INFO:tasks.rados.rados.0.smithi026.stdout:1549: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:24:12.206 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 0 (ObjNum 33 snap 0 seq_num 519200000) dirty dne 2024-09-13T14:24:12.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:11 smithi145 ceph-mon[89145]: pgmap v134: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 op/s 2024-09-13T14:24:12.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:11 smithi145 ceph-mon[89145]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:24:12.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.207 INFO:tasks.rados.rados.0.smithi026.stdout:1547: finishing write tid 2 to smithi02683976-20 2024-09-13T14:24:12.207 INFO:tasks.rados.rados.0.smithi026.stdout:1547: finishing write tid 3 to smithi02683976-20 2024-09-13T14:24:12.208 INFO:tasks.rados.rados.0.smithi026.stdout:1547: finishing write tid 4 to smithi02683976-20 2024-09-13T14:24:12.208 INFO:tasks.rados.rados.0.smithi026.stdout:1547: finishing write tid 5 to smithi02683976-20 2024-09-13T14:24:12.208 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 739 (ObjNum 519 snap 142 seq_num 519) dirty exists 2024-09-13T14:24:12.208 INFO:tasks.rados.rados.0.smithi026.stdout:1547: left oid 20 (ObjNum 519 snap 142 seq_num 519) 2024-09-13T14:24:12.222 INFO:tasks.rados.rados.0.smithi026.stdout:1548: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:24:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[97022]: pgmap v134: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 op/s 2024-09-13T14:24:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[97022]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:24:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[99461]: pgmap v134: 121 pgs: 121 active+clean; 130 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 op/s 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[99461]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.960 INFO:tasks.rados.rados.0.smithi026.stdout:1543: done (6 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1544: done (5 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1547: done (4 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1548: done (3 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1549: done (2 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1550: done (1 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1551: done (0 left) 2024-09-13T14:24:12.961 INFO:tasks.rados.rados.0.smithi026.stdout:1552: delete oid 30 current snap is 143 2024-09-13T14:24:12.965 INFO:tasks.rados.rados.0.smithi026.stdout:1552: done (0 left) 2024-09-13T14:24:12.965 INFO:tasks.rados.rados.0.smithi026.stdout:1553: snap_create 2024-09-13T14:24:13.905 INFO:tasks.rados.rados.0.smithi026.stdout:1553: done (0 left) 2024-09-13T14:24:13.906 INFO:tasks.rados.rados.0.smithi026.stdout:1554: read oid 13 snap -1 2024-09-13T14:24:13.906 INFO:tasks.rados.rados.0.smithi026.stdout:1554: expect deleted 2024-09-13T14:24:13.906 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 144 2024-09-13T14:24:13.906 INFO:tasks.rados.rados.0.smithi026.stdout:1555: seq_num 521 ranges {4063232=729088,4792320=720896,5513216=81920} 2024-09-13T14:24:13.931 INFO:tasks.rados.rados.0.smithi026.stdout:1555: writing smithi02683976-32 from 4063232 to 4792320 tid 1 2024-09-13T14:24:13.936 INFO:tasks.rados.rados.0.smithi026.stdout:1555: writing smithi02683976-32 from 4792320 to 5513216 tid 2 2024-09-13T14:24:13.937 INFO:tasks.rados.rados.0.smithi026.stdout:1555: writing smithi02683976-32 from 5513216 to 5595136 tid 3 2024-09-13T14:24:13.937 INFO:tasks.rados.rados.0.smithi026.stdout:1556: copy_from oid 3 from oid 14 current snap is 144 2024-09-13T14:24:13.937 INFO:tasks.rados.rados.0.smithi026.stdout:1554: done (2 left) 2024-09-13T14:24:13.937 INFO:tasks.rados.rados.0.smithi026.stdout:1557: snap_create 2024-09-13T14:24:13.940 INFO:tasks.rados.rados.0.smithi026.stdout:1555: finishing write tid 1 to smithi02683976-32 2024-09-13T14:24:13.940 INFO:tasks.rados.rados.0.smithi026.stdout:1555: finishing write tid 2 to smithi02683976-32 2024-09-13T14:24:13.941 INFO:tasks.rados.rados.0.smithi026.stdout:1555: finishing write tid 3 to smithi02683976-32 2024-09-13T14:24:13.941 INFO:tasks.rados.rados.0.smithi026.stdout:1555: finishing write tid 4 to smithi02683976-32 2024-09-13T14:24:13.945 INFO:tasks.rados.rados.0.smithi026.stdout:1555: finishing write tid 5 to smithi02683976-32 2024-09-13T14:24:13.945 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 720 (ObjNum 521 snap 144 seq_num 521) dirty exists 2024-09-13T14:24:13.946 INFO:tasks.rados.rados.0.smithi026.stdout:1555: left oid 32 (ObjNum 521 snap 144 seq_num 521) 2024-09-13T14:24:13.961 INFO:tasks.rados.rados.0.smithi026.stdout:1556: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:24:13.962 INFO:tasks.rados.rados.0.smithi026.stdout:1556: finishing copy_from to smithi02683976-3 2024-09-13T14:24:13.962 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 594 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:24:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:13 smithi145 ceph-mon[89145]: pgmap v136: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:13 smithi145 ceph-mon[89145]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-13T14:24:14.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:13 smithi145 ceph-mon[89145]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:24:14.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:13 smithi145 ceph-mon[89145]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:24:14.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:14.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[99461]: pgmap v136: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:14.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[99461]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-13T14:24:14.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[99461]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[99461]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[97022]: pgmap v136: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[97022]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[97022]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:24:14.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:13 smithi026 ceph-mon[97022]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:24:14.906 INFO:tasks.rados.rados.0.smithi026.stdout:1555: done (2 left) 2024-09-13T14:24:14.907 INFO:tasks.rados.rados.0.smithi026.stdout:1556: done (1 left) 2024-09-13T14:24:14.907 INFO:tasks.rados.rados.0.smithi026.stdout:1557: done (0 left) 2024-09-13T14:24:14.907 INFO:tasks.rados.rados.0.smithi026.stdout:1558: delete oid 33 current snap is 145 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1558: done (0 left) 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1559: read oid 12 snap -1 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1559: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1560: read oid 34 snap -1 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1560: expect deleted 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1561: read oid 11 snap 144 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1561: expect deleted 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1562: read oid 32 snap -1 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1562: expect (ObjNum 521 snap 144 seq_num 521) 2024-09-13T14:24:14.912 INFO:tasks.rados.rados.0.smithi026.stdout:1563: snap_create 2024-09-13T14:24:14.935 INFO:tasks.rados.rados.0.smithi026.stdout:1562: expect (ObjNum 521 snap 144 seq_num 521) 2024-09-13T14:24:15.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:15.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:15.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[99461]: Upgrade: osd.1 is safe to restart 2024-09-13T14:24:15.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[99461]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:24:15.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:15.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:15.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[97022]: Upgrade: osd.1 is safe to restart 2024-09-13T14:24:15.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:14 smithi026 ceph-mon[97022]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:24:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:24:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:14 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:24:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:14 smithi145 ceph-mon[89145]: Upgrade: osd.1 is safe to restart 2024-09-13T14:24:15.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:14 smithi145 ceph-mon[89145]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:24:15.223 INFO:tasks.rados.rados.0.smithi026.stdout:1559: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1559: done (4 left) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1560: done (3 left) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1561: done (2 left) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1562: done (1 left) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1563: done (0 left) 2024-09-13T14:24:15.909 INFO:tasks.rados.rados.0.smithi026.stdout:1564: delete oid 35 current snap is 146 2024-09-13T14:24:15.911 INFO:tasks.rados.rados.0.smithi026.stdout:1564: done (0 left) 2024-09-13T14:24:15.911 INFO:tasks.rados.rados.0.smithi026.stdout:1565: rollback oid 37 current snap is 146 2024-09-13T14:24:15.912 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 144 2024-09-13T14:24:15.912 INFO:tasks.rados.rados.0.smithi026.stdout:1566: snap_create 2024-09-13T14:24:15.916 INFO:tasks.rados.rados.0.smithi026.stdout:1565: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:24:15.917 INFO:tasks.rados.rados.0.smithi026.stdout:1565: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:24:15.917 INFO:tasks.rados.rados.0.smithi026.stdout:1565: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:24:15.917 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 364 (ObjNum 441 snap 122 seq_num 441) dirty exists 2024-09-13T14:24:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-mon[89145]: pgmap v139: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:16.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:16.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:24:16.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:16.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-mon[89145]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:24:16.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:16.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[97022]: pgmap v139: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[97022]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:24:16.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[99461]: pgmap v139: 121 pgs: 121 active+clean; 141 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.5 MiB/s wr, 22 op/s 2024-09-13T14:24:16.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:16.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:24:16.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:16.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:15 smithi026 ceph-mon[99461]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:24:16.909 INFO:tasks.rados.rados.0.smithi026.stdout:1565: done (1 left) 2024-09-13T14:24:16.910 INFO:tasks.rados.rados.0.smithi026.stdout:1566: done (0 left) 2024-09-13T14:24:16.910 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 147 2024-09-13T14:24:16.910 INFO:tasks.rados.rados.0.smithi026.stdout:1567: seq_num 524 ranges {2039808=786432,2826240=778240,3604480=24576} 2024-09-13T14:24:16.924 INFO:tasks.rados.rados.0.smithi026.stdout:1567: writing smithi02683976-38 from 2039808 to 2826240 tid 1 2024-09-13T14:24:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:1567: writing smithi02683976-38 from 2826240 to 3604480 tid 2 2024-09-13T14:24:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:1567: writing smithi02683976-38 from 3604480 to 3629056 tid 3 2024-09-13T14:24:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:1568: rollback oid 50 current snap is 147 2024-09-13T14:24:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 142 2024-09-13T14:24:16.930 INFO:tasks.rados.rados.0.smithi026.stdout:1569: setattr oid 41 current snap is 147 2024-09-13T14:24:16.932 INFO:tasks.rados.rados.0.smithi026.stdout:1570: rollback oid 7 current snap is 147 2024-09-13T14:24:16.932 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 7 to 141 2024-09-13T14:24:16.932 INFO:tasks.rados.rados.0.smithi026.stdout:1571: snap_remove snap 146 2024-09-13T14:24:17.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[99461]: Upgrade: Updating osd.1 2024-09-13T14:24:17.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[99461]: Deploying daemon osd.1 on smithi026 2024-09-13T14:24:17.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[99461]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:24:17.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[97022]: Upgrade: Updating osd.1 2024-09-13T14:24:17.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[97022]: Deploying daemon osd.1 on smithi026 2024-09-13T14:24:17.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:16 smithi026 ceph-mon[97022]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:24:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:16 smithi145 ceph-mon[89145]: Upgrade: Updating osd.1 2024-09-13T14:24:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:16 smithi145 ceph-mon[89145]: Deploying daemon osd.1 on smithi026 2024-09-13T14:24:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:16 smithi145 ceph-mon[89145]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:24:17.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:17.911 INFO:tasks.rados.rados.0.smithi026.stdout:1571: done (4 left) 2024-09-13T14:24:17.911 INFO:tasks.rados.rados.0.smithi026.stdout:1572: rollback oid 14 current snap is 147 2024-09-13T14:24:17.911 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 141 2024-09-13T14:24:17.911 INFO:tasks.rados.rados.0.smithi026.stdout:1567: finishing write tid 1 to smithi02683976-38 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1567: finishing write tid 2 to smithi02683976-38 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1570: finishing rollback tid 1 to smithi02683976-7 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 0 (ObjNum 161 snap 0 seq_num 519448720) dirty dne 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1567: finishing write tid 3 to smithi02683976-38 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1567: finishing write tid 4 to smithi02683976-38 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1568: finishing rollback tid 0 to smithi02683976-50 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1567: finishing write tid 5 to smithi02683976-38 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 725 (ObjNum 524 snap 147 seq_num 524) dirty exists 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1567: left oid 38 (ObjNum 524 snap 147 seq_num 524) 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1568: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:1568: finishing rollback tid 2 to smithi02683976-50 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 807 (ObjNum 501 snap 138 seq_num 501) dirty exists 2024-09-13T14:24:17.912 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 788 (ObjNum 12288 snap 4 seq_num 304) dirty exists 2024-09-13T14:24:17.913 INFO:tasks.rados.rados.0.smithi026.stdout:1567: done (4 left) 2024-09-13T14:24:17.913 INFO:tasks.rados.rados.0.smithi026.stdout:1568: done (3 left) 2024-09-13T14:24:17.913 INFO:tasks.rados.rados.0.smithi026.stdout:1569: done (2 left) 2024-09-13T14:24:17.913 INFO:tasks.rados.rados.0.smithi026.stdout:1570: done (1 left) 2024-09-13T14:24:17.913 INFO:tasks.rados.rados.0.smithi026.stdout:1573: delete oid 31 current snap is 147 2024-09-13T14:24:17.917 INFO:tasks.rados.rados.0.smithi026.stdout:1573: done (1 left) 2024-09-13T14:24:17.917 INFO:tasks.rados.rados.0.smithi026.stdout:1574: snap_remove snap 142 2024-09-13T14:24:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:17 smithi026 ceph-mon[97022]: pgmap v142: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:17 smithi026 ceph-mon[97022]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:24:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:17 smithi026 ceph-mon[99461]: pgmap v142: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:17 smithi026 ceph-mon[99461]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:24:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:17 smithi145 ceph-mon[89145]: pgmap v142: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:17 smithi145 ceph-mon[89145]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1574: done (1 left) 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1575: read oid 23 snap 144 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1575: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1576: read oid 12 snap -1 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1576: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1577: setattr oid 16 current snap is 147 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1572: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1572: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:1572: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:24:18.913 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 729 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1572: done (3 left) 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1578: read oid 37 snap -1 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1578: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1579: read oid 2 snap -1 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1579: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:24:18.914 INFO:tasks.rados.rados.0.smithi026.stdout:1580: copy_from oid 32 from oid 21 current snap is 147 2024-09-13T14:24:18.915 INFO:tasks.rados.rados.0.smithi026.stdout:1581: snap_remove snap 145 2024-09-13T14:24:19.103 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:18 smithi026 systemd[1]: Stopping Ceph osd.1 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:24:19.415 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[49641]: 2024-09-13T14:24:19.101+0000 7f8e60364640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:24:19.415 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[49641]: 2024-09-13T14:24:19.101+0000 7f8e60364640 -1 osd.1 366 *** Got signal Terminated *** 2024-09-13T14:24:19.415 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[49641]: 2024-09-13T14:24:19.101+0000 7f8e60364640 -1 osd.1 366 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:24:19.921 INFO:tasks.rados.rados.0.smithi026.stdout:1581: done (6 left) 2024-09-13T14:24:19.921 INFO:tasks.rados.rados.0.smithi026.stdout:1582: delete oid 36 current snap is 147 2024-09-13T14:24:19.922 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 741 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:19.922 INFO:tasks.rados.rados.0.smithi026.stdout:1579: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:24:20.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[99461]: pgmap v145: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:20.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[99461]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:24:20.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[99461]: osd.1 marked itself down and dead 2024-09-13T14:24:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[97022]: pgmap v145: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[97022]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:24:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:19 smithi026 ceph-mon[97022]: osd.1 marked itself down and dead 2024-09-13T14:24:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:19 smithi145 ceph-mon[89145]: pgmap v145: 121 pgs: 121 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-13T14:24:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:19 smithi145 ceph-mon[89145]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:24:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:19 smithi145 ceph-mon[89145]: osd.1 marked itself down and dead 2024-09-13T14:24:20.266 INFO:tasks.rados.rados.0.smithi026.stdout:1576: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:20.628 INFO:tasks.rados.rados.0.smithi026.stdout:1578: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:20.665 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:20 smithi026 podman[109903]: 2024-09-13 14:24:20.412011866 +0000 UTC m=+1.414523207 container died 8d2e66aacf4c4135b0a8f6556df9e26f3e18bbc3d00ee9b70c00c798293f30d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:24:20.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:21.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:20 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:21.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:20 smithi026 ceph-mon[97022]: osdmap e367: 8 total, 7 up, 8 in 2024-09-13T14:24:21.136 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:20 smithi026 podman[109903]: 2024-09-13 14:24:20.883723683 +0000 UTC m=+1.886235025 container cleanup 8d2e66aacf4c4135b0a8f6556df9e26f3e18bbc3d00ee9b70c00c798293f30d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:24:21.136 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:20 smithi026 bash[109903]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:24:21.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:20 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:21.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:20 smithi026 ceph-mon[99461]: osdmap e367: 8 total, 7 up, 8 in 2024-09-13T14:24:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:20 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:20 smithi145 ceph-mon[89145]: osdmap e367: 8 total, 7 up, 8 in 2024-09-13T14:24:21.266 INFO:tasks.rados.rados.0.smithi026.stdout:1575: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:24:21.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:21] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-13T14:24:21.415 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[109923]: 2024-09-13 14:24:21.135288218 +0000 UTC m=+0.724828126 container remove 8d2e66aacf4c4135b0a8f6556df9e26f3e18bbc3d00ee9b70c00c798293f30d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1580: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1580: finishing copy_from to smithi02683976-32 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 459 (ObjNum 268 snap 69 seq_num 268) dirty exists 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1575: done (6 left) 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1576: done (5 left) 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1577: done (4 left) 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1578: done (3 left) 2024-09-13T14:24:21.700 INFO:tasks.rados.rados.0.smithi026.stdout:1579: done (2 left) 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1580: done (1 left) 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1582: done (0 left) 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1583: read oid 24 snap -1 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1583: expect (ObjNum 510 snap 142 seq_num 510) 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1584: setattr oid 18 current snap is 147 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1585: copy_from oid 2 from oid 47 current snap is 147 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1586: delete oid 45 current snap is 147 2024-09-13T14:24:21.701 INFO:tasks.rados.rados.0.smithi026.stdout:1586: done (3 left) 2024-09-13T14:24:21.702 INFO:tasks.rados.rados.0.smithi026.stdout:1587: read oid 12 snap -1 2024-09-13T14:24:21.702 INFO:tasks.rados.rados.0.smithi026.stdout:1587: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:21.702 INFO:tasks.rados.rados.0.smithi026.stdout:1588: snap_remove snap 141 2024-09-13T14:24:21.765 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.369856194 +0000 UTC m=+0.016653198 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:21.765 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.536628971 +0000 UTC m=+0.183425974 container create 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux ) 2024-09-13T14:24:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:1588: done (4 left) 2024-09-13T14:24:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:1589: read oid 17 snap -1 2024-09-13T14:24:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:1589: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:24:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 147 2024-09-13T14:24:21.932 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 741 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:24:21.933 INFO:tasks.rados.rados.0.smithi026.stdout:1583: expect (ObjNum 510 snap 142 seq_num 510) 2024-09-13T14:24:22.020 INFO:tasks.rados.rados.0.smithi026.stdout:1585: finishing copy_from racing read to smithi02683976-2 2024-09-13T14:24:22.021 INFO:tasks.rados.rados.0.smithi026.stdout:1585: finishing copy_from to smithi02683976-2 2024-09-13T14:24:22.021 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 305 (ObjNum 476 snap 126 seq_num 476) dirty exists 2024-09-13T14:24:22.021 INFO:tasks.rados.rados.0.smithi026.stdout:1587: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:22.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:21 smithi026 ceph-mon[99461]: pgmap v148: 121 pgs: 16 stale+active+clean, 105 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:22.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:21 smithi026 ceph-mon[99461]: osdmap e368: 8 total, 7 up, 8 in 2024-09-13T14:24:22.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:21 smithi026 ceph-mon[97022]: pgmap v148: 121 pgs: 16 stale+active+clean, 105 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:22.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:21 smithi026 ceph-mon[97022]: osdmap e368: 8 total, 7 up, 8 in 2024-09-13T14:24:22.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.764687415 +0000 UTC m=+0.411484421 container init 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True) 2024-09-13T14:24:22.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.770015906 +0000 UTC m=+0.416812912 container start 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T14:24:22.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.86442803 +0000 UTC m=+0.511225034 container attach 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:24:22.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:21 smithi026 podman[110013]: 2024-09-13 14:24:21.964670928 +0000 UTC m=+0.611467942 container died 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T14:24:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:21 smithi145 ceph-mon[89145]: pgmap v148: 121 pgs: 16 stale+active+clean, 105 active+clean; 149 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:22.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:21 smithi145 ceph-mon[89145]: osdmap e368: 8 total, 7 up, 8 in 2024-09-13T14:24:22.374 INFO:tasks.rados.rados.0.smithi026.stdout:1589: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:24:22.636 INFO:tasks.rados.rados.0.smithi026.stdout:1590: seq_num 528 ranges {4243456=450560,4694016=761856,5455872=442368} 2024-09-13T14:24:22.662 INFO:tasks.rados.rados.0.smithi026.stdout:1590: writing smithi02683976-42 from 4243456 to 4694016 tid 1 2024-09-13T14:24:22.666 INFO:tasks.rados.rados.0.smithi026.stdout:1590: writing smithi02683976-42 from 4694016 to 5455872 tid 2 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1590: writing smithi02683976-42 from 5455872 to 5898240 tid 3 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1583: done (5 left) 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1584: done (4 left) 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1585: done (3 left) 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1587: done (2 left) 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1589: done (1 left) 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 147 2024-09-13T14:24:22.670 INFO:tasks.rados.rados.0.smithi026.stdout:1591: seq_num 529 ranges {2277376=466944,2744320=647168,3391488=532480,3923968=24576} 2024-09-13T14:24:22.684 INFO:tasks.rados.rados.0.smithi026.stdout:1591: writing smithi02683976-9 from 2277376 to 2744320 tid 1 2024-09-13T14:24:22.688 INFO:tasks.rados.rados.0.smithi026.stdout:1591: writing smithi02683976-9 from 2744320 to 3391488 tid 2 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1591: writing smithi02683976-9 from 3391488 to 3923968 tid 3 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1591: writing smithi02683976-9 from 3923968 to 3948544 tid 4 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1592: snap_create 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: finishing write tid 1 to smithi02683976-42 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: finishing write tid 2 to smithi02683976-42 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: finishing write tid 3 to smithi02683976-42 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: finishing write tid 4 to smithi02683976-42 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: finishing write tid 5 to smithi02683976-42 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 368 (ObjNum 528 snap 147 seq_num 528) dirty exists 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1590: left oid 42 (ObjNum 528 snap 147 seq_num 528) 2024-09-13T14:24:22.692 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 1 to smithi02683976-9 2024-09-13T14:24:22.693 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 2 to smithi02683976-9 2024-09-13T14:24:22.694 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 3 to smithi02683976-9 2024-09-13T14:24:22.695 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 4 to smithi02683976-9 2024-09-13T14:24:22.695 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 5 to smithi02683976-9 2024-09-13T14:24:22.696 INFO:tasks.rados.rados.0.smithi026.stdout:1591: finishing write tid 6 to smithi02683976-9 2024-09-13T14:24:22.696 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 821 (ObjNum 529 snap 147 seq_num 529) dirty exists 2024-09-13T14:24:22.696 INFO:tasks.rados.rados.0.smithi026.stdout:1591: left oid 9 (ObjNum 529 snap 147 seq_num 529) 2024-09-13T14:24:22.938 INFO:tasks.rados.rados.0.smithi026.stdout:1590: done (2 left) 2024-09-13T14:24:22.938 INFO:tasks.rados.rados.0.smithi026.stdout:1591: done (1 left) 2024-09-13T14:24:22.938 INFO:tasks.rados.rados.0.smithi026.stdout:1592: done (0 left) 2024-09-13T14:24:22.938 INFO:tasks.rados.rados.0.smithi026.stdout:1593: read oid 26 snap -1 2024-09-13T14:24:22.939 INFO:tasks.rados.rados.0.smithi026.stdout:1593: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:22.939 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 148 2024-09-13T14:24:22.939 INFO:tasks.rados.rados.0.smithi026.stdout:1594: seq_num 530 ranges {2293760=483328,2777088=532480,3309568=671744} 2024-09-13T14:24:22.953 INFO:tasks.rados.rados.0.smithi026.stdout:1594: writing smithi02683976-28 from 2293760 to 2777088 tid 1 2024-09-13T14:24:22.957 INFO:tasks.rados.rados.0.smithi026.stdout:1594: writing smithi02683976-28 from 2777088 to 3309568 tid 2 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1594: writing smithi02683976-28 from 3309568 to 3981312 tid 3 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1595: copy_from oid 35 from oid 40 current snap is 148 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1594: finishing write tid 1 to smithi02683976-28 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1594: finishing write tid 2 to smithi02683976-28 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1596: rmattr oid 44 current snap is 148 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1596: done (3 left) 2024-09-13T14:24:22.962 INFO:tasks.rados.rados.0.smithi026.stdout:1597: snap_remove snap 144 2024-09-13T14:24:23.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:22 smithi026 ceph-mon[97022]: osdmap e369: 8 total, 7 up, 8 in 2024-09-13T14:24:23.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:22 smithi026 ceph-mon[99461]: osdmap e369: 8 total, 7 up, 8 in 2024-09-13T14:24:23.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:22 smithi026 podman[110056]: 2024-09-13 14:24:22.80188736 +0000 UTC m=+0.831276045 container remove 2790c4bacd7cdffb90bccc776bf36fe2fb272698f85eeec678e437b71b8ee268 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:24:23.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:22 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T14:24:23.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:22 smithi026 systemd[1]: Stopped Ceph osd.1 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:24:23.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:22 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.1.service: Consumed 13.450s CPU time. 2024-09-13T14:24:23.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 systemd[1]: Starting Ceph osd.1 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:24:23.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:22 smithi145 ceph-mon[89145]: osdmap e369: 8 total, 7 up, 8 in 2024-09-13T14:24:23.573 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 podman[110157]: 2024-09-13 14:24:23.143787688 +0000 UTC m=+0.016177112 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:23.573 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 podman[110157]: 2024-09-13 14:24:23.261772789 +0000 UTC m=+0.134162200 container create 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T14:24:23.573 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 podman[110157]: 2024-09-13 14:24:23.481083161 +0000 UTC m=+0.353472573 container init 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-09-13T14:24:23.573 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 podman[110157]: 2024-09-13 14:24:23.486005893 +0000 UTC m=+0.358395312 container start 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:24:23.573 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 podman[110157]: 2024-09-13 14:24:23.572780972 +0000 UTC m=+0.445170383 container attach 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T14:24:23.908 INFO:tasks.rados.rados.0.smithi026.stdout:1597: done (3 left) 2024-09-13T14:24:23.908 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 148 2024-09-13T14:24:23.908 INFO:tasks.rados.rados.0.smithi026.stdout:1593: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:23.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:24:23.916 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:23 smithi026 bash[110157]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:24:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:23 smithi145 ceph-mon[89145]: pgmap v151: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:23 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 48/279 objects degraded (17.204%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:23 smithi145 ceph-mon[89145]: osdmap e370: 8 total, 7 up, 8 in 2024-09-13T14:24:24.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:23 smithi145 ceph-mon[89145]: osdmap e371: 8 total, 7 up, 8 in 2024-09-13T14:24:24.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:24.280 INFO:tasks.rados.rados.0.smithi026.stdout:1594: finishing write tid 3 to smithi02683976-28 2024-09-13T14:24:24.280 INFO:tasks.rados.rados.0.smithi026.stdout:1594: finishing write tid 4 to smithi02683976-28 2024-09-13T14:24:24.280 INFO:tasks.rados.rados.0.smithi026.stdout:1598: seq_num 531 ranges {827392=499712,1327104=417792,1744896=548864,2293760=237568} 2024-09-13T14:24:24.287 INFO:tasks.rados.rados.0.smithi026.stdout:1598: writing smithi02683976-19 from 827392 to 1327104 tid 1 2024-09-13T14:24:24.289 INFO:tasks.rados.rados.0.smithi026.stdout:1598: writing smithi02683976-19 from 1327104 to 1744896 tid 2 2024-09-13T14:24:24.292 INFO:tasks.rados.rados.0.smithi026.stdout:1598: writing smithi02683976-19 from 1744896 to 2293760 tid 3 2024-09-13T14:24:24.294 INFO:tasks.rados.rados.0.smithi026.stdout:1598: writing smithi02683976-19 from 2293760 to 2531328 tid 4 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1593: done (3 left) 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1599: copy_from oid 2 from oid 31 current snap is 148 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1595: finishing copy_from to smithi02683976-35 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1595: got expected ENOENT (src dne) 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1594: finishing write tid 5 to smithi02683976-28 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 804 (ObjNum 530 snap 148 seq_num 530) dirty exists 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1594: left oid 28 (ObjNum 530 snap 148 seq_num 530) 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1595: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1594: done (3 left) 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1595: done (2 left) 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:1600: rollback oid 33 current snap is 148 2024-09-13T14:24:24.295 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 143 2024-09-13T14:24:24.296 INFO:tasks.rados.rados.0.smithi026.stdout:1601: rollback oid 16 current snap is 148 2024-09-13T14:24:24.296 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 143 2024-09-13T14:24:24.296 INFO:tasks.rados.rados.0.smithi026.stdout:1602: snap_remove snap 147 2024-09-13T14:24:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[99461]: pgmap v151: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 48/279 objects degraded (17.204%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[99461]: osdmap e370: 8 total, 7 up, 8 in 2024-09-13T14:24:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[99461]: osdmap e371: 8 total, 7 up, 8 in 2024-09-13T14:24:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[97022]: pgmap v151: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 48/279 objects degraded (17.204%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[97022]: osdmap e370: 8 total, 7 up, 8 in 2024-09-13T14:24:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:23 smithi026 ceph-mon[97022]: osdmap e371: 8 total, 7 up, 8 in 2024-09-13T14:24:24.416 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:24:24.416 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 bash[110157]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:24:24.416 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 bash[110157]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 bash[110157]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 bash[110157]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate[110174]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 bash[110157]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:24:24.417 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 podman[110157]: 2024-09-13 14:24:24.054086453 +0000 UTC m=+0.926475875 container died 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1599: finishing copy_from to smithi02683976-2 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1599: got expected ENOENT (src dne) 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1599: finishing copy_from racing read to smithi02683976-2 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 1 to smithi02683976-19 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 2 to smithi02683976-19 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 3 to smithi02683976-19 2024-09-13T14:24:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 4 to smithi02683976-19 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1600: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1600: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 901 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 5 to smithi02683976-19 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1601: finishing rollback tid 0 to smithi02683976-16 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1601: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1598: finishing write tid 6 to smithi02683976-19 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 747 (ObjNum 531 snap 148 seq_num 531) dirty exists 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1598: left oid 19 (ObjNum 531 snap 148 seq_num 531) 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1601: finishing rollback tid 2 to smithi02683976-16 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 751 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1598: done (4 left) 2024-09-13T14:24:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:1599: done (3 left) 2024-09-13T14:24:24.949 INFO:tasks.rados.rados.0.smithi026.stdout:1600: done (2 left) 2024-09-13T14:24:24.949 INFO:tasks.rados.rados.0.smithi026.stdout:1601: done (1 left) 2024-09-13T14:24:24.949 INFO:tasks.rados.rados.0.smithi026.stdout:1602: done (0 left) 2024-09-13T14:24:24.949 INFO:tasks.rados.rados.0.smithi026.stdout:1603: snap_remove snap 143 2024-09-13T14:24:25.245 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:24 smithi026 podman[110304]: 2024-09-13 14:24:24.918874353 +0000 UTC m=+0.855770110 container remove 451f6d9840d757453a7fc8b775670c0b6731b5ed50a07af67a11cf19bd92ef60 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:24:25.245 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 podman[110371]: 2024-09-13 14:24:25.044471246 +0000 UTC m=+0.016866072 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:25.496 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 podman[110371]: 2024-09-13 14:24:25.245021137 +0000 UTC m=+0.217416027 container create 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux ) 2024-09-13T14:24:25.496 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 podman[110371]: 2024-09-13 14:24:25.447628808 +0000 UTC m=+0.420023641 container init 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:24:25.496 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 podman[110371]: 2024-09-13 14:24:25.453117061 +0000 UTC m=+0.425511882 container start 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T14:24:25.496 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-osd[110409]: -- 172.21.15.26:0/1639963505 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bbbf4b5a40 con 0x55bbbf658400 2024-09-13T14:24:25.917 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 bash[110371]: 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa 2024-09-13T14:24:25.917 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:25 smithi026 systemd[1]: Started Ceph osd.1 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:24:25.954 INFO:tasks.rados.rados.0.smithi026.stdout:1603: done (0 left) 2024-09-13T14:24:25.954 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 148 2024-09-13T14:24:25.954 INFO:tasks.rados.rados.0.smithi026.stdout:1604: seq_num 532 ranges {2482176=516096,2998272=696320,3694592=507904} 2024-09-13T14:24:25.971 INFO:tasks.rados.rados.0.smithi026.stdout:1604: writing smithi02683976-24 from 2482176 to 2998272 tid 1 2024-09-13T14:24:25.976 INFO:tasks.rados.rados.0.smithi026.stdout:1604: writing smithi02683976-24 from 2998272 to 3694592 tid 2 2024-09-13T14:24:25.979 INFO:tasks.rados.rados.0.smithi026.stdout:1604: writing smithi02683976-24 from 3694592 to 4202496 tid 3 2024-09-13T14:24:25.979 INFO:tasks.rados.rados.0.smithi026.stdout:1605: read oid 5 snap -1 2024-09-13T14:24:25.980 INFO:tasks.rados.rados.0.smithi026.stdout:1605: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:25.980 INFO:tasks.rados.rados.0.smithi026.stdout:1606: read oid 33 snap -1 2024-09-13T14:24:25.980 INFO:tasks.rados.rados.0.smithi026.stdout:1606: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:24:25.980 INFO:tasks.rados.rados.0.smithi026.stdout:1604: finishing write tid 1 to smithi02683976-24 2024-09-13T14:24:25.981 INFO:tasks.rados.rados.0.smithi026.stdout:1607: read oid 37 snap -1 2024-09-13T14:24:25.981 INFO:tasks.rados.rados.0.smithi026.stdout:1607: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:25.981 INFO:tasks.rados.rados.0.smithi026.stdout:1604: finishing write tid 2 to smithi02683976-24 2024-09-13T14:24:25.981 INFO:tasks.rados.rados.0.smithi026.stdout:1608: snap_create 2024-09-13T14:24:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:1604: finishing write tid 3 to smithi02683976-24 2024-09-13T14:24:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:1604: finishing write tid 4 to smithi02683976-24 2024-09-13T14:24:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:1604: finishing write tid 5 to smithi02683976-24 2024-09-13T14:24:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 611 (ObjNum 532 snap 148 seq_num 532) dirty exists 2024-09-13T14:24:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:1604: left oid 24 (ObjNum 532 snap 148 seq_num 532) 2024-09-13T14:24:26.004 INFO:tasks.rados.rados.0.smithi026.stdout:1605: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:26.134 INFO:tasks.rados.rados.0.smithi026.stdout:1607: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-mon[89145]: pgmap v154: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-mon[89145]: osdmap e372: 8 total, 7 up, 8 in 2024-09-13T14:24:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:26.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[99461]: pgmap v154: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:26.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[99461]: osdmap e372: 8 total, 7 up, 8 in 2024-09-13T14:24:26.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[97022]: pgmap v154: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 157 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 48/279 objects degraded (17.204%) 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[97022]: osdmap e372: 8 total, 7 up, 8 in 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:26.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:25 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:26.768 INFO:tasks.rados.rados.0.smithi026.stdout:1606: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1604: done (4 left) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1605: done (3 left) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1606: done (2 left) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1607: done (1 left) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1608: done (0 left) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1609: rollback oid 1 current snap is 149 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 148 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1610: read oid 32 snap -1 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1610: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:24:26.960 INFO:tasks.rados.rados.0.smithi026.stdout:1611: setattr oid 12 current snap is 149 2024-09-13T14:24:26.961 INFO:tasks.rados.rados.0.smithi026.stdout:1609: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:24:26.962 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 0 (ObjNum 465 snap 0 seq_num 1126582865) dirty dne 2024-09-13T14:24:26.962 INFO:tasks.rados.rados.0.smithi026.stdout:1609: done (2 left) 2024-09-13T14:24:26.962 INFO:tasks.rados.rados.0.smithi026.stdout:1612: read oid 49 snap 148 2024-09-13T14:24:26.962 INFO:tasks.rados.rados.0.smithi026.stdout:1612: expect (ObjNum 455 snap 124 seq_num 455) 2024-09-13T14:24:26.962 INFO:tasks.rados.rados.0.smithi026.stdout:1613: setattr oid 13 current snap is 149 2024-09-13T14:24:26.964 INFO:tasks.rados.rados.0.smithi026.stdout:1614: read oid 47 snap 148 2024-09-13T14:24:26.964 INFO:tasks.rados.rados.0.smithi026.stdout:1614: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:24:26.965 INFO:tasks.rados.rados.0.smithi026.stdout:1615: read oid 17 snap -1 2024-09-13T14:24:26.965 INFO:tasks.rados.rados.0.smithi026.stdout:1615: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:24:26.965 INFO:tasks.rados.rados.0.smithi026.stdout:1616: copy_from oid 9 from oid 5 current snap is 149 2024-09-13T14:24:26.965 INFO:tasks.rados.rados.0.smithi026.stdout:1617: setattr oid 45 current snap is 149 2024-09-13T14:24:26.967 INFO:tasks.rados.rados.0.smithi026.stdout:1618: delete oid 26 current snap is 149 2024-09-13T14:24:26.972 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 752 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-13T14:24:26.973 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 765 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1611: done (8 left) 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1613: done (7 left) 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1618: done (6 left) 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1619: read oid 14 snap 148 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1619: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:24:26.974 INFO:tasks.rados.rados.0.smithi026.stdout:1620: read oid 20 snap -1 2024-09-13T14:24:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1620: expect (ObjNum 519 snap 142 seq_num 519) 2024-09-13T14:24:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1621: copy_from oid 41 from oid 44 current snap is 149 2024-09-13T14:24:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 149 2024-09-13T14:24:26.975 INFO:tasks.rados.rados.0.smithi026.stdout:1622: seq_num 534 ranges {2760704=548864,3309568=516096,3825664=688128} 2024-09-13T14:24:26.995 INFO:tasks.rados.rados.0.smithi026.stdout:1622: writing smithi02683976-2 from 2760704 to 3309568 tid 1 2024-09-13T14:24:26.999 INFO:tasks.rados.rados.0.smithi026.stdout:1622: writing smithi02683976-2 from 3309568 to 3825664 tid 2 2024-09-13T14:24:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:1622: writing smithi02683976-2 from 3825664 to 4513792 tid 3 2024-09-13T14:24:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 292 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:24:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:1610: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:24:27.183 INFO:tasks.rados.rados.0.smithi026.stdout:1614: expect (ObjNum 476 snap 126 seq_num 476) 2024-09-13T14:24:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:26 smithi145 ceph-mon[89145]: osdmap e373: 8 total, 7 up, 8 in 2024-09-13T14:24:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:27.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:27.238 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:27 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:24:27.090+0000 7f85fad38640 -1 Falling back to public interface 2024-09-13T14:24:27.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[99461]: osdmap e373: 8 total, 7 up, 8 in 2024-09-13T14:24:27.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:27.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:27.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[97022]: osdmap e373: 8 total, 7 up, 8 in 2024-09-13T14:24:27.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:27.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:27.331 INFO:tasks.rados.rados.0.smithi026.stdout:1616: finishing copy_from racing read to smithi02683976-9 2024-09-13T14:24:27.331 INFO:tasks.rados.rados.0.smithi026.stdout:1616: finishing copy_from to smithi02683976-9 2024-09-13T14:24:27.332 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 796 (ObjNum 467 snap 125 seq_num 467) dirty exists 2024-09-13T14:24:27.332 INFO:tasks.rados.rados.0.smithi026.stdout:1615: expect (ObjNum 433 snap 122 seq_num 433) 2024-09-13T14:24:27.595 INFO:tasks.rados.rados.0.smithi026.stdout:1622: finishing write tid 1 to smithi02683976-2 2024-09-13T14:24:27.595 INFO:tasks.rados.rados.0.smithi026.stdout:1612: expect (ObjNum 455 snap 124 seq_num 455) 2024-09-13T14:24:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:1622: finishing write tid 2 to smithi02683976-2 2024-09-13T14:24:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:1620: expect (ObjNum 519 snap 142 seq_num 519) 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:1622: finishing write tid 3 to smithi02683976-2 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:1622: finishing write tid 4 to smithi02683976-2 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:1622: finishing write tid 5 to smithi02683976-2 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 615 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:1622: left oid 2 (ObjNum 534 snap 149 seq_num 534) 2024-09-13T14:24:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:1619: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:24:28.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:27 smithi026 ceph-mon[99461]: pgmap v157: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 150 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 52/258 objects degraded (20.155%) 2024-09-13T14:24:28.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:27 smithi026 ceph-mon[99461]: osdmap e374: 8 total, 7 up, 8 in 2024-09-13T14:24:28.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:27 smithi026 ceph-mon[97022]: pgmap v157: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 150 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 52/258 objects degraded (20.155%) 2024-09-13T14:24:28.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:27 smithi026 ceph-mon[97022]: osdmap e374: 8 total, 7 up, 8 in 2024-09-13T14:24:28.439 INFO:tasks.rados.rados.0.smithi026.stdout:1621: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:24:28.439 INFO:tasks.rados.rados.0.smithi026.stdout:1621: finishing copy_from to smithi02683976-41 2024-09-13T14:24:28.439 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 450 (ObjNum 508 snap 140 seq_num 508) dirty exists 2024-09-13T14:24:28.439 INFO:tasks.rados.rados.0.smithi026.stdout:1610: done (9 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1612: done (8 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1614: done (7 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1615: done (6 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1616: done (5 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1617: done (4 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1619: done (3 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1620: done (2 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1621: done (1 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1622: done (0 left) 2024-09-13T14:24:28.440 INFO:tasks.rados.rados.0.smithi026.stdout:1623: snap_create 2024-09-13T14:24:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:27 smithi145 ceph-mon[89145]: pgmap v157: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 150 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 52/258 objects degraded (20.155%) 2024-09-13T14:24:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:27 smithi145 ceph-mon[89145]: osdmap e374: 8 total, 7 up, 8 in 2024-09-13T14:24:28.965 INFO:tasks.rados.rados.0.smithi026.stdout:1623: done (0 left) 2024-09-13T14:24:28.965 INFO:tasks.rados.rados.0.smithi026.stdout:1624: copy_from oid 50 from oid 10 current snap is 150 2024-09-13T14:24:28.965 INFO:tasks.rados.rados.0.smithi026.stdout:1625: snap_remove snap 148 2024-09-13T14:24:29.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:28 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 52/258 objects degraded (20.155%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:29.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:28 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 52/258 objects degraded (20.155%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:28 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 52/258 objects degraded (20.155%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:1625: done (1 left) 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:1626: copy_from oid 5 from oid 2 current snap is 150 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:1624: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:1624: finishing copy_from to smithi02683976-50 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 796 (ObjNum 467 snap 125 seq_num 467) dirty exists 2024-09-13T14:24:29.968 INFO:tasks.rados.rados.0.smithi026.stdout:1624: done (1 left) 2024-09-13T14:24:29.969 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 150 2024-09-13T14:24:29.969 INFO:tasks.rados.rados.0.smithi026.stdout:1627: seq_num 535 ranges {0=565248,565248=802816,1368064=401408} 2024-09-13T14:24:29.973 INFO:tasks.rados.rados.0.smithi026.stdout:1627: writing smithi02683976-7 from 0 to 565248 tid 1 2024-09-13T14:24:29.978 INFO:tasks.rados.rados.0.smithi026.stdout:1627: writing smithi02683976-7 from 565248 to 1368064 tid 2 2024-09-13T14:24:29.981 INFO:tasks.rados.rados.0.smithi026.stdout:1627: writing smithi02683976-7 from 1368064 to 1769472 tid 3 2024-09-13T14:24:29.981 INFO:tasks.rados.rados.0.smithi026.stdout:1628: snap_create 2024-09-13T14:24:29.981 INFO:tasks.rados.rados.0.smithi026.stdout:1627: finishing write tid 1 to smithi02683976-7 2024-09-13T14:24:29.982 INFO:tasks.rados.rados.0.smithi026.stdout:1627: finishing write tid 2 to smithi02683976-7 2024-09-13T14:24:29.983 INFO:tasks.rados.rados.0.smithi026.stdout:1627: finishing write tid 3 to smithi02683976-7 2024-09-13T14:24:29.983 INFO:tasks.rados.rados.0.smithi026.stdout:1627: finishing write tid 4 to smithi02683976-7 2024-09-13T14:24:29.985 INFO:tasks.rados.rados.0.smithi026.stdout:1627: finishing write tid 5 to smithi02683976-7 2024-09-13T14:24:29.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 518 (ObjNum 535 snap 150 seq_num 535) dirty exists 2024-09-13T14:24:29.985 INFO:tasks.rados.rados.0.smithi026.stdout:1627: left oid 7 (ObjNum 535 snap 150 seq_num 535) 2024-09-13T14:24:30.001 INFO:tasks.rados.rados.0.smithi026.stdout:1626: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:24:30.003 INFO:tasks.rados.rados.0.smithi026.stdout:1626: finishing copy_from to smithi02683976-5 2024-09-13T14:24:30.003 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 615 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:24:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[99461]: pgmap v159: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[99461]: osdmap e375: 8 total, 7 up, 8 in 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[97022]: pgmap v159: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[97022]: osdmap e375: 8 total, 7 up, 8 in 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:29 smithi145 ceph-mon[89145]: pgmap v159: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:30.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:29 smithi145 ceph-mon[89145]: osdmap e375: 8 total, 7 up, 8 in 2024-09-13T14:24:30.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:30.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:30.710 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:24:30.565+0000 7f85fad38640 -1 osd.1 366 log_to_monitors true 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1626: done (2 left) 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1627: done (1 left) 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1628: done (0 left) 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1629: read oid 9 snap -1 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1629: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1630: read oid 50 snap -1 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:1630: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:30.982 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 151 2024-09-13T14:24:30.983 INFO:tasks.rados.rados.0.smithi026.stdout:1631: seq_num 536 ranges {4096000=581632,4677632=688128,5365760=516096} 2024-09-13T14:24:31.009 INFO:tasks.rados.rados.0.smithi026.stdout:1631: writing smithi02683976-17 from 4096000 to 4677632 tid 1 2024-09-13T14:24:31.013 INFO:tasks.rados.rados.0.smithi026.stdout:1631: writing smithi02683976-17 from 4677632 to 5365760 tid 2 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:1631: writing smithi02683976-17 from 5365760 to 5881856 tid 3 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:1632: rmattr oid 44 current snap is 151 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:1632: done (3 left) 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:1633: rollback oid 38 current snap is 151 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 150 2024-09-13T14:24:31.016 INFO:tasks.rados.rados.0.smithi026.stdout:1629: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:31.145 INFO:tasks.rados.rados.0.smithi026.stdout:1630: expect (ObjNum 467 snap 125 seq_num 467) 2024-09-13T14:24:31.276 INFO:tasks.rados.rados.0.smithi026.stdout:1631: finishing write tid 1 to smithi02683976-17 2024-09-13T14:24:31.276 INFO:tasks.rados.rados.0.smithi026.stdout:1631: finishing write tid 2 to smithi02683976-17 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1631: finishing write tid 3 to smithi02683976-17 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1631: finishing write tid 4 to smithi02683976-17 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1631: finishing write tid 5 to smithi02683976-17 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 792 (ObjNum 536 snap 151 seq_num 536) dirty exists 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1631: left oid 17 (ObjNum 536 snap 151 seq_num 536) 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1633: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1633: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1633: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 745 (ObjNum 524 snap 147 seq_num 524) dirty exists 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1629: done (3 left) 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1630: done (2 left) 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1631: done (1 left) 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1633: done (0 left) 2024-09-13T14:24:31.277 INFO:tasks.rados.rados.0.smithi026.stdout:1634: read oid 26 snap 150 2024-09-13T14:24:31.278 INFO:tasks.rados.rados.0.smithi026.stdout:1634: expect deleted 2024-09-13T14:24:31.278 INFO:tasks.rados.rados.0.smithi026.stdout:1635: copy_from oid 19 from oid 2 current snap is 151 2024-09-13T14:24:31.278 INFO:tasks.rados.rados.0.smithi026.stdout:1636: snap_create 2024-09-13T14:24:31.298 INFO:tasks.rados.rados.0.smithi026.stdout:1635: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:24:31.300 INFO:tasks.rados.rados.0.smithi026.stdout:1635: finishing copy_from to smithi02683976-19 2024-09-13T14:24:31.300 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 615 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:24:31.351 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:24:31.003+0000 7f85f2ae2640 -1 osd.1 366 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:24:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[99461]: osdmap e376: 8 total, 7 up, 8 in 2024-09-13T14:24:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[99461]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[99461]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[97022]: osdmap e376: 8 total, 7 up, 8 in 2024-09-13T14:24:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[97022]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:30 smithi026 ceph-mon[97022]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-mon[89145]: osdmap e376: 8 total, 7 up, 8 in 2024-09-13T14:24:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-mon[89145]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:30 smithi145 ceph-mon[89145]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:24:31.646 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:31] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.51.0" 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1634: done (2 left) 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1635: done (1 left) 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1636: done (0 left) 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1637: read oid 34 snap -1 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1637: expect deleted 2024-09-13T14:24:31.991 INFO:tasks.rados.rados.0.smithi026.stdout:1638: setattr oid 5 current snap is 152 2024-09-13T14:24:31.993 INFO:tasks.rados.rados.0.smithi026.stdout:1639: snap_create 2024-09-13T14:24:32.011 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 919 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[97022]: pgmap v162: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[97022]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[97022]: osdmap e377: 8 total, 7 up, 8 in 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[97022]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[97022]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[99461]: pgmap v162: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[99461]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:24:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[99461]: osdmap e377: 8 total, 7 up, 8 in 2024-09-13T14:24:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[99461]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:31 smithi026 ceph-mon[99461]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:31 smithi145 ceph-mon[89145]: pgmap v162: 121 pgs: 34 active+undersized, 11 active+undersized+degraded, 76 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 12 op/s; 53/261 objects degraded (20.307%) 2024-09-13T14:24:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:31 smithi145 ceph-mon[89145]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:24:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:31 smithi145 ceph-mon[89145]: osdmap e377: 8 total, 7 up, 8 in 2024-09-13T14:24:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:31 smithi145 ceph-mon[89145]: from='osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:31 smithi145 ceph-mon[89145]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:24:32.996 INFO:tasks.rados.rados.0.smithi026.stdout:1637: done (2 left) 2024-09-13T14:24:32.996 INFO:tasks.rados.rados.0.smithi026.stdout:1638: done (1 left) 2024-09-13T14:24:32.996 INFO:tasks.rados.rados.0.smithi026.stdout:1639: done (0 left) 2024-09-13T14:24:32.996 INFO:tasks.rados.rados.0.smithi026.stdout:1640: delete oid 6 current snap is 153 2024-09-13T14:24:33.017 INFO:tasks.rados.rados.0.smithi026.stdout:1640: done (0 left) 2024-09-13T14:24:33.017 INFO:tasks.rados.rados.0.smithi026.stdout:1641: snap_create 2024-09-13T14:24:33.080 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:24:33.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[97022]: osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333] boot 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[97022]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[99461]: osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333] boot 2024-09-13T14:24:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[99461]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:24:33.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:32 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:24:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:32 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:32 smithi145 ceph-mon[89145]: osd.1 [v2:172.21.15.26:6810/2254834333,v1:172.21.15.26:6811/2254834333] boot 2024-09-13T14:24:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:32 smithi145 ceph-mon[89145]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:24:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:32 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:24:33.727 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:24:33.727 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 4s ago 8m 19.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:24:33.727 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (2m) 80s ago 7m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:24:33.727 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (9m) 4s ago 9m 15.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:24:33.727 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (9m) 80s ago 9m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:24:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (2m) 80s ago 12m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:24:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (2m) 4s ago 14m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:24:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (2m) 4s ago 14m 50.3M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:24:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (85s) 80s ago 13m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:24:33.728 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (97s) 4s ago 13m 42.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 4s ago 7m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (2m) 80s ago 7m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (43s) 4s ago 12m 92.1M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8s) 4s ago 12m 11.5M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (11m) 4s ago 11m 179M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b7165e08f421 2024-09-13T14:24:33.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (11m) 4s ago 11m 108M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:24:33.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (10m) 80s ago 10m 171M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:24:33.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (10m) 80s ago 10m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:24:33.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (10m) 80s ago 10m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:24:33.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (9m) 80s ago 9m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:24:33.730 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (2m) 80s ago 8m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:24:33.731 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:24:33.608+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:24:34.000 INFO:tasks.rados.rados.0.smithi026.stdout:1641: done (0 left) 2024-09-13T14:24:34.001 INFO:tasks.rados.rados.0.smithi026.stdout:1642: snap_remove snap 149 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:24:34.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 8, 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:24:34.079 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:24:34.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: pgmap v165: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:34.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:24:34.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:34.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: pgmap v165: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:34.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:33 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:24:34.378 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:24:34.379 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "7/20 daemons upgraded", 2024-09-13T14:24:34.379 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:24:34.379 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:24:34.379 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:24:34.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:34.150Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:24:34.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:34.151Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: pgmap v165: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:34.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:34.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:33 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:35.007 INFO:tasks.rados.rados.0.smithi026.stdout:1642: done (0 left) 2024-09-13T14:24:35.007 INFO:tasks.rados.rados.0.smithi026.stdout:1643: delete oid 32 current snap is 154 2024-09-13T14:24:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1643: done (0 left) 2024-09-13T14:24:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:1644: delete oid 21 current snap is 154 2024-09-13T14:24:35.023 INFO:tasks.rados.rados.0.smithi026.stdout:1644: done (0 left) 2024-09-13T14:24:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:1645: snap_remove snap 152 2024-09-13T14:24:35.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3176018924' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[99461]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:24:35.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:24:35.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3176018924' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:35.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:35 smithi026 ceph-mon[97022]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:24:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:24:35.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3176018924' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:35.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-mon[89145]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:36.004 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:36.004 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:36.010 INFO:tasks.rados.rados.0.smithi026.stdout:1645: done (0 left) 2024-09-13T14:24:36.010 INFO:tasks.rados.rados.0.smithi026.stdout:1646: snap_remove snap 153 2024-09-13T14:24:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:36 smithi026 ceph-mon[97022]: pgmap v168: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:36 smithi026 ceph-mon[97022]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:24:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:36 smithi026 ceph-mon[99461]: pgmap v168: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:36 smithi026 ceph-mon[99461]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:24:36.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:36 smithi145 ceph-mon[89145]: pgmap v168: 121 pgs: 6 peering, 28 active+undersized, 11 active+undersized+degraded, 76 active+clean; 173 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 28 op/s; 52/285 objects degraded (18.246%) 2024-09-13T14:24:36.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:36 smithi145 ceph-mon[89145]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:24:37.018 INFO:tasks.rados.rados.0.smithi026.stdout:1646: done (0 left) 2024-09-13T14:24:37.018 INFO:tasks.rados.rados.0.smithi026.stdout:1647: copy_from oid 11 from oid 16 current snap is 154 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1648: rollback oid 40 current snap is 154 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 40 to 150 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1649: read oid 17 snap -1 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1649: expect (ObjNum 536 snap 151 seq_num 536) 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1650: read oid 44 snap -1 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1650: expect (ObjNum 508 snap 140 seq_num 508) 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1651: delete oid 9 current snap is 154 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:1648: finishing rollback tid 1 to smithi02683976-40 2024-09-13T14:24:37.019 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 0 (ObjNum 145 snap 0 seq_num 519058200) dirty dne 2024-09-13T14:24:37.031 INFO:tasks.rados.rados.0.smithi026.stdout:1648: done (4 left) 2024-09-13T14:24:37.031 INFO:tasks.rados.rados.0.smithi026.stdout:1651: done (3 left) 2024-09-13T14:24:37.031 INFO:tasks.rados.rados.0.smithi026.stdout:1652: rollback oid 50 current snap is 154 2024-09-13T14:24:37.031 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 151 2024-09-13T14:24:37.031 INFO:tasks.rados.rados.0.smithi026.stdout:1653: snap_create 2024-09-13T14:24:37.034 INFO:tasks.rados.rados.0.smithi026.stdout:1652: finishing rollback tid 0 to smithi02683976-50 2024-09-13T14:24:37.037 INFO:tasks.rados.rados.0.smithi026.stdout:1652: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:24:37.037 INFO:tasks.rados.rados.0.smithi026.stdout:1652: finishing rollback tid 2 to smithi02683976-50 2024-09-13T14:24:37.037 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 859 (ObjNum 467 snap 125 seq_num 467) dirty exists 2024-09-13T14:24:37.055 INFO:tasks.rados.rados.0.smithi026.stdout:1647: finishing copy_from racing read to smithi02683976-11 2024-09-13T14:24:37.058 INFO:tasks.rados.rados.0.smithi026.stdout:1649: expect (ObjNum 536 snap 151 seq_num 536) 2024-09-13T14:24:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:37 smithi145 ceph-mon[89145]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:24:37.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:36 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:37.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:37 smithi026 ceph-mon[99461]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:24:37.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:37 smithi026 ceph-mon[97022]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:24:37.438 INFO:tasks.rados.rados.0.smithi026.stdout:1647: finishing copy_from to smithi02683976-11 2024-09-13T14:24:37.438 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 751 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:37.438 INFO:tasks.rados.rados.0.smithi026.stdout:1650: expect (ObjNum 508 snap 140 seq_num 508) 2024-09-13T14:24:38.114 INFO:tasks.rados.rados.0.smithi026.stdout:1647: done (4 left) 2024-09-13T14:24:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:1649: done (3 left) 2024-09-13T14:24:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:1650: done (2 left) 2024-09-13T14:24:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:1652: done (1 left) 2024-09-13T14:24:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:1653: done (0 left) 2024-09-13T14:24:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:1654: snap_create 2024-09-13T14:24:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[97022]: pgmap v171: 121 pgs: 6 peering, 115 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-13T14:24:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded) 2024-09-13T14:24:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:24:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[97022]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:24:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[99461]: pgmap v171: 121 pgs: 6 peering, 115 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-13T14:24:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded) 2024-09-13T14:24:38.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:24:38.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:38 smithi026 ceph-mon[99461]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:24:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:38 smithi145 ceph-mon[89145]: pgmap v171: 121 pgs: 6 peering, 115 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-13T14:24:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:38 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/285 objects degraded (18.246%), 11 pgs degraded) 2024-09-13T14:24:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:38 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:24:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:38 smithi145 ceph-mon[89145]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:24:39.029 INFO:tasks.rados.rados.0.smithi026.stdout:1654: done (0 left) 2024-09-13T14:24:39.030 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 156 2024-09-13T14:24:39.030 INFO:tasks.rados.rados.0.smithi026.stdout:1655: seq_num 541 ranges {4882432=663552,5545984=557056,6103040=475136,6578176=172032} 2024-09-13T14:24:39.059 INFO:tasks.rados.rados.0.smithi026.stdout:1655: writing smithi02683976-14 from 4882432 to 5545984 tid 1 2024-09-13T14:24:39.063 INFO:tasks.rados.rados.0.smithi026.stdout:1655: writing smithi02683976-14 from 5545984 to 6103040 tid 2 2024-09-13T14:24:39.066 INFO:tasks.rados.rados.0.smithi026.stdout:1655: writing smithi02683976-14 from 6103040 to 6578176 tid 3 2024-09-13T14:24:39.067 INFO:tasks.rados.rados.0.smithi026.stdout:1655: writing smithi02683976-14 from 6578176 to 6750208 tid 4 2024-09-13T14:24:39.067 INFO:tasks.rados.rados.0.smithi026.stdout:1656: setattr oid 16 current snap is 156 2024-09-13T14:24:39.068 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 1 to smithi02683976-14 2024-09-13T14:24:39.068 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 2 to smithi02683976-14 2024-09-13T14:24:39.069 INFO:tasks.rados.rados.0.smithi026.stdout:1657: snap_remove snap 154 2024-09-13T14:24:39.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:39 smithi026 ceph-mon[99461]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:24:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:39 smithi026 ceph-mon[97022]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:24:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:39 smithi145 ceph-mon[89145]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:24:40.037 INFO:tasks.rados.rados.0.smithi026.stdout:1657: done (2 left) 2024-09-13T14:24:40.037 INFO:tasks.rados.rados.0.smithi026.stdout:1658: rmattr oid 8 current snap is 156 2024-09-13T14:24:40.037 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 3 to smithi02683976-14 2024-09-13T14:24:40.037 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 4 to smithi02683976-14 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 5 to smithi02683976-14 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1655: finishing write tid 6 to smithi02683976-14 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 771 (ObjNum 541 snap 156 seq_num 541) dirty exists 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1655: left oid 14 (ObjNum 541 snap 156 seq_num 541) 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 768 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1655: done (2 left) 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1656: done (1 left) 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1659: read oid 39 snap 150 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1659: expect (ObjNum 408 snap 116 seq_num 408) 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1660: read oid 21 snap 150 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1660: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:24:40.038 INFO:tasks.rados.rados.0.smithi026.stdout:1661: snap_remove snap 151 2024-09-13T14:24:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:40 smithi026 ceph-mon[99461]: pgmap v174: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-13T14:24:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:40 smithi026 ceph-mon[99461]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:24:40.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:40 smithi026 ceph-mon[97022]: pgmap v174: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-13T14:24:40.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:40 smithi026 ceph-mon[97022]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:24:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:40 smithi145 ceph-mon[89145]: pgmap v174: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-13T14:24:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:40 smithi145 ceph-mon[89145]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:24:40.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:41.040 INFO:tasks.rados.rados.0.smithi026.stdout:1661: done (3 left) 2024-09-13T14:24:41.040 INFO:tasks.rados.rados.0.smithi026.stdout:1662: rollback oid 47 current snap is 156 2024-09-13T14:24:41.040 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 155 2024-09-13T14:24:41.041 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 583 (ObjNum 498 snap 138 seq_num 498) dirty exists 2024-09-13T14:24:41.041 INFO:tasks.rados.rados.0.smithi026.stdout:1659: expect (ObjNum 408 snap 116 seq_num 408) 2024-09-13T14:24:41.101 INFO:tasks.rados.rados.0.smithi026.stdout:1660: expect (ObjNum 268 snap 69 seq_num 268) 2024-09-13T14:24:41.278 INFO:tasks.rados.rados.0.smithi026.stdout:1662: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:24:41.278 INFO:tasks.rados.rados.0.smithi026.stdout:1662: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1662: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 313 (ObjNum 476 snap 126 seq_num 476) dirty exists 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1658: done (3 left) 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1659: done (2 left) 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1660: done (1 left) 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1662: done (0 left) 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1663: copy_from oid 19 from oid 31 current snap is 156 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1664: copy_from oid 3 from oid 21 current snap is 156 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1665: read oid 12 snap -1 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:1665: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:41.279 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 156 2024-09-13T14:24:41.280 INFO:tasks.rados.rados.0.smithi026.stdout:1666: seq_num 542 ranges {2572288=679936,3252224=442368,3694592=679936,4374528=81920} 2024-09-13T14:24:41.295 INFO:tasks.rados.rados.0.smithi026.stdout:1666: writing smithi02683976-46 from 2572288 to 3252224 tid 1 2024-09-13T14:24:41.298 INFO:tasks.rados.rados.0.smithi026.stdout:1666: writing smithi02683976-46 from 3252224 to 3694592 tid 2 2024-09-13T14:24:41.301 INFO:tasks.rados.rados.0.smithi026.stdout:1666: writing smithi02683976-46 from 3694592 to 4374528 tid 3 2024-09-13T14:24:41.302 INFO:tasks.rados.rados.0.smithi026.stdout:1666: writing smithi02683976-46 from 4374528 to 4456448 tid 4 2024-09-13T14:24:41.302 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 156 2024-09-13T14:24:41.302 INFO:tasks.rados.rados.0.smithi026.stdout:1664: finishing copy_from to smithi02683976-3 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1664: got expected ENOENT (src dne) 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1664: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1663: finishing copy_from to smithi02683976-19 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1663: got expected ENOENT (src dne) 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1663: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:24:41.303 INFO:tasks.rados.rados.0.smithi026.stdout:1667: seq_num 543 ranges {0=704512,704512=729088,1433600=475136} 2024-09-13T14:24:41.307 INFO:tasks.rados.rados.0.smithi026.stdout:1667: writing smithi02683976-29 from 0 to 704512 tid 1 2024-09-13T14:24:41.311 INFO:tasks.rados.rados.0.smithi026.stdout:1667: writing smithi02683976-29 from 704512 to 1433600 tid 2 2024-09-13T14:24:41.314 INFO:tasks.rados.rados.0.smithi026.stdout:1667: writing smithi02683976-29 from 1433600 to 1908736 tid 3 2024-09-13T14:24:41.314 INFO:tasks.rados.rados.0.smithi026.stdout:1663: done (4 left) 2024-09-13T14:24:41.314 INFO:tasks.rados.rados.0.smithi026.stdout:1664: done (3 left) 2024-09-13T14:24:41.314 INFO:tasks.rados.rados.0.smithi026.stdout:1668: snap_remove snap 155 2024-09-13T14:24:41.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:41 smithi026 ceph-mon[99461]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:24:41.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:41 smithi026 ceph-mon[97022]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:24:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:41 smithi145 ceph-mon[89145]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:24:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:41] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-13T14:24:42.162 INFO:tasks.rados.rados.0.smithi026.stdout:1668: done (3 left) 2024-09-13T14:24:42.162 INFO:tasks.rados.rados.0.smithi026.stdout:1669: rollback oid 19 current snap is 156 2024-09-13T14:24:42.162 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 150 2024-09-13T14:24:42.162 INFO:tasks.rados.rados.0.smithi026.stdout:1665: expect (ObjNum 485 snap 132 seq_num 485) 2024-09-13T14:24:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[97022]: pgmap v177: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s, 0 objects/s recovering 2024-09-13T14:24:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[97022]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:24:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[99461]: pgmap v177: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s, 0 objects/s recovering 2024-09-13T14:24:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[99461]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:24:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:42 smithi145 ceph-mon[89145]: pgmap v177: 121 pgs: 121 active+clean; 149 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s, 0 objects/s recovering 2024-09-13T14:24:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:42 smithi145 ceph-mon[89145]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:24:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:42.522 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 1 to smithi02683976-46 2024-09-13T14:24:42.522 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 2 to smithi02683976-46 2024-09-13T14:24:42.522 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 3 to smithi02683976-46 2024-09-13T14:24:42.522 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 4 to smithi02683976-46 2024-09-13T14:24:42.522 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 5 to smithi02683976-46 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1665: done (3 left) 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1670: copy_from oid 10 from oid 45 current snap is 156 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1666: finishing write tid 6 to smithi02683976-46 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 455 (ObjNum 542 snap 156 seq_num 542) dirty exists 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1666: left oid 46 (ObjNum 542 snap 156 seq_num 542) 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1667: finishing write tid 1 to smithi02683976-29 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1667: finishing write tid 2 to smithi02683976-29 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1667: finishing write tid 3 to smithi02683976-29 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1667: finishing write tid 4 to smithi02683976-29 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:1667: finishing write tid 5 to smithi02683976-29 2024-09-13T14:24:42.523 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 777 (ObjNum 543 snap 156 seq_num 543) dirty exists 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1667: left oid 29 (ObjNum 543 snap 156 seq_num 543) 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1669: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1669: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1669: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 784 (ObjNum 531 snap 148 seq_num 531) dirty exists 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1666: done (3 left) 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1667: done (2 left) 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1669: done (1 left) 2024-09-13T14:24:42.524 INFO:tasks.rados.rados.0.smithi026.stdout:1671: setattr oid 12 current snap is 156 2024-09-13T14:24:42.525 INFO:tasks.rados.rados.0.smithi026.stdout:1672: copy_from oid 38 from oid 31 current snap is 156 2024-09-13T14:24:42.525 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 156 2024-09-13T14:24:42.525 INFO:tasks.rados.rados.0.smithi026.stdout:1673: seq_num 544 ranges {4202496=720896,4923392=614400,5537792=589824} 2024-09-13T14:24:42.551 INFO:tasks.rados.rados.0.smithi026.stdout:1673: writing smithi02683976-24 from 4202496 to 4923392 tid 1 2024-09-13T14:24:42.554 INFO:tasks.rados.rados.0.smithi026.stdout:1673: writing smithi02683976-24 from 4923392 to 5537792 tid 2 2024-09-13T14:24:42.558 INFO:tasks.rados.rados.0.smithi026.stdout:1673: writing smithi02683976-24 from 5537792 to 6127616 tid 3 2024-09-13T14:24:42.558 INFO:tasks.rados.rados.0.smithi026.stdout:1670: finishing copy_from racing read to smithi02683976-10 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:1672: finishing copy_from to smithi02683976-38 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:1672: got expected ENOENT (src dne) 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:1672: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:1670: finishing copy_from to smithi02683976-10 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 292 (ObjNum 145 snap 0 seq_num 518968320) dirty exists 2024-09-13T14:24:42.559 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 767 (ObjNum 485 snap 132 seq_num 485) dirty exists 2024-09-13T14:24:42.560 INFO:tasks.rados.rados.0.smithi026.stdout:1673: finishing write tid 1 to smithi02683976-24 2024-09-13T14:24:42.560 INFO:tasks.rados.rados.0.smithi026.stdout:1673: finishing write tid 2 to smithi02683976-24 2024-09-13T14:24:42.560 INFO:tasks.rados.rados.0.smithi026.stdout:1670: done (3 left) 2024-09-13T14:24:42.560 INFO:tasks.rados.rados.0.smithi026.stdout:1671: done (2 left) 2024-09-13T14:24:42.561 INFO:tasks.rados.rados.0.smithi026.stdout:1672: done (1 left) 2024-09-13T14:24:42.561 INFO:tasks.rados.rados.0.smithi026.stdout:1674: snap_remove snap 150 2024-09-13T14:24:43.170 INFO:tasks.rados.rados.0.smithi026.stdout:1674: done (1 left) 2024-09-13T14:24:43.170 INFO:tasks.rados.rados.0.smithi026.stdout:1675: copy_from oid 41 from oid 34 current snap is 156 2024-09-13T14:24:43.170 INFO:tasks.rados.rados.0.smithi026.stdout:1673: finishing write tid 3 to smithi02683976-24 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:1673: finishing write tid 4 to smithi02683976-24 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:1673: finishing write tid 5 to smithi02683976-24 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 619 (ObjNum 544 snap 156 seq_num 544) dirty exists 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:1673: left oid 24 (ObjNum 544 snap 156 seq_num 544) 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:1673: done (1 left) 2024-09-13T14:24:43.171 INFO:tasks.rados.rados.0.smithi026.stdout:1676: delete oid 50 current snap is 156 2024-09-13T14:24:43.177 INFO:tasks.rados.rados.0.smithi026.stdout:1675: finishing copy_from to smithi02683976-41 2024-09-13T14:24:43.177 INFO:tasks.rados.rados.0.smithi026.stdout:1675: got expected ENOENT (src dne) 2024-09-13T14:24:43.177 INFO:tasks.rados.rados.0.smithi026.stdout:1675: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:1675: done (1 left) 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:1676: done (0 left) 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:1677: rollback oid 41 current snap is 156 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:1677: done (0 left) 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 156 2024-09-13T14:24:43.181 INFO:tasks.rados.rados.0.smithi026.stdout:1678: seq_num 546 ranges {5898240=753664,6651904=778240,7430144=425984} 2024-09-13T14:24:43.217 INFO:tasks.rados.rados.0.smithi026.stdout:1678: writing smithi02683976-42 from 5898240 to 6651904 tid 1 2024-09-13T14:24:43.222 INFO:tasks.rados.rados.0.smithi026.stdout:1678: writing smithi02683976-42 from 6651904 to 7430144 tid 2 2024-09-13T14:24:43.225 INFO:tasks.rados.rados.0.smithi026.stdout:1678: writing smithi02683976-42 from 7430144 to 7856128 tid 3 2024-09-13T14:24:43.225 INFO:tasks.rados.rados.0.smithi026.stdout:1679: copy_from oid 48 from oid 43 current snap is 156 2024-09-13T14:24:43.225 INFO:tasks.rados.rados.0.smithi026.stdout:1678: finishing write tid 1 to smithi02683976-42 2024-09-13T14:24:43.225 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 156 2024-09-13T14:24:43.225 INFO:tasks.rados.rados.0.smithi026.stdout:1680: seq_num 547 ranges {5029888=770048,5799936=663552,6463488=466944,6930432=73728} 2024-09-13T14:24:43.257 INFO:tasks.rados.rados.0.smithi026.stdout:1680: writing smithi02683976-18 from 5029888 to 5799936 tid 1 2024-09-13T14:24:43.261 INFO:tasks.rados.rados.0.smithi026.stdout:1680: writing smithi02683976-18 from 5799936 to 6463488 tid 2 2024-09-13T14:24:43.263 INFO:tasks.rados.rados.0.smithi026.stdout:1680: writing smithi02683976-18 from 6463488 to 6930432 tid 3 2024-09-13T14:24:43.264 INFO:tasks.rados.rados.0.smithi026.stdout:1680: writing smithi02683976-18 from 6930432 to 7004160 tid 4 2024-09-13T14:24:43.264 INFO:tasks.rados.rados.0.smithi026.stdout:1681: snap_create 2024-09-13T14:24:43.264 INFO:tasks.rados.rados.0.smithi026.stdout:1678: finishing write tid 2 to smithi02683976-42 2024-09-13T14:24:43.264 INFO:tasks.rados.rados.0.smithi026.stdout:1678: finishing write tid 3 to smithi02683976-42 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1678: finishing write tid 4 to smithi02683976-42 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1678: finishing write tid 5 to smithi02683976-42 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 372 (ObjNum 546 snap 156 seq_num 546) dirty exists 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1678: left oid 42 (ObjNum 546 snap 156 seq_num 546) 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1679: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1679: finishing copy_from to smithi02683976-48 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 713 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 1 to smithi02683976-18 2024-09-13T14:24:43.265 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 2 to smithi02683976-18 2024-09-13T14:24:43.267 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 3 to smithi02683976-18 2024-09-13T14:24:43.267 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 4 to smithi02683976-18 2024-09-13T14:24:43.267 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 5 to smithi02683976-18 2024-09-13T14:24:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:1680: finishing write tid 6 to smithi02683976-18 2024-09-13T14:24:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 790 (ObjNum 547 snap 156 seq_num 547) dirty exists 2024-09-13T14:24:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:1680: left oid 18 (ObjNum 547 snap 156 seq_num 547) 2024-09-13T14:24:43.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:43 smithi026 ceph-mon[97022]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:24:43.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:43 smithi026 ceph-mon[99461]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:24:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:43 smithi145 ceph-mon[89145]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:1678: done (3 left) 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:1679: done (2 left) 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:1680: done (1 left) 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:1681: done (0 left) 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 157 2024-09-13T14:24:43.913 INFO:tasks.rados.rados.0.smithi026.stdout:1682: seq_num 548 ranges {2760704=786432,3547136=598016,4145152=606208} 2024-09-13T14:24:43.932 INFO:tasks.rados.rados.0.smithi026.stdout:1682: writing smithi02683976-47 from 2760704 to 3547136 tid 1 2024-09-13T14:24:43.936 INFO:tasks.rados.rados.0.smithi026.stdout:1682: writing smithi02683976-47 from 3547136 to 4145152 tid 2 2024-09-13T14:24:43.941 INFO:tasks.rados.rados.0.smithi026.stdout:1682: writing smithi02683976-47 from 4145152 to 4751360 tid 3 2024-09-13T14:24:43.941 INFO:tasks.rados.rados.0.smithi026.stdout:1683: read oid 37 snap -1 2024-09-13T14:24:43.941 INFO:tasks.rados.rados.0.smithi026.stdout:1683: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:43.941 INFO:tasks.rados.rados.0.smithi026.stdout:1684: read oid 9 snap -1 2024-09-13T14:24:43.941 INFO:tasks.rados.rados.0.smithi026.stdout:1684: expect deleted 2024-09-13T14:24:43.942 INFO:tasks.rados.rados.0.smithi026.stdout:1685: setattr oid 26 current snap is 157 2024-09-13T14:24:43.942 INFO:tasks.rados.rados.0.smithi026.stdout:1682: finishing write tid 1 to smithi02683976-47 2024-09-13T14:24:43.942 INFO:tasks.rados.rados.0.smithi026.stdout:1682: finishing write tid 2 to smithi02683976-47 2024-09-13T14:24:43.943 INFO:tasks.rados.rados.0.smithi026.stdout:1684: done (3 left) 2024-09-13T14:24:43.943 INFO:tasks.rados.rados.0.smithi026.stdout:1686: rmattr oid 43 current snap is 157 2024-09-13T14:24:43.943 INFO:tasks.rados.rados.0.smithi026.stdout:1686: done (3 left) 2024-09-13T14:24:43.943 INFO:tasks.rados.rados.0.smithi026.stdout:1687: snap_create 2024-09-13T14:24:43.943 INFO:tasks.rados.rados.0.smithi026.stdout:1682: finishing write tid 3 to smithi02683976-47 2024-09-13T14:24:43.944 INFO:tasks.rados.rados.0.smithi026.stdout:1682: finishing write tid 4 to smithi02683976-47 2024-09-13T14:24:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 768 (ObjNum 721 snap 0 seq_num 1574039376) dirty exists 2024-09-13T14:24:43.951 INFO:tasks.rados.rados.0.smithi026.stdout:1682: finishing write tid 5 to smithi02683976-47 2024-09-13T14:24:43.951 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 717 (ObjNum 548 snap 157 seq_num 548) dirty exists 2024-09-13T14:24:43.951 INFO:tasks.rados.rados.0.smithi026.stdout:1682: left oid 47 (ObjNum 548 snap 157 seq_num 548) 2024-09-13T14:24:43.964 INFO:tasks.rados.rados.0.smithi026.stdout:1683: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:44.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[97022]: pgmap v180: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:44.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[97022]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:24:44.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[97022]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:24:44.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[99461]: pgmap v180: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:44.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[99461]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:24:44.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:44 smithi026 ceph-mon[99461]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:24:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:44 smithi145 ceph-mon[89145]: pgmap v180: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:44 smithi145 ceph-mon[89145]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:24:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:44 smithi145 ceph-mon[89145]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:24:44.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:44.913 INFO:tasks.rados.rados.0.smithi026.stdout:1682: done (3 left) 2024-09-13T14:24:44.913 INFO:tasks.rados.rados.0.smithi026.stdout:1683: done (2 left) 2024-09-13T14:24:44.913 INFO:tasks.rados.rados.0.smithi026.stdout:1685: done (1 left) 2024-09-13T14:24:44.913 INFO:tasks.rados.rados.0.smithi026.stdout:1687: done (0 left) 2024-09-13T14:24:44.913 INFO:tasks.rados.rados.0.smithi026.stdout:1688: snap_remove snap 156 2024-09-13T14:24:45.915 INFO:tasks.rados.rados.0.smithi026.stdout:1688: done (0 left) 2024-09-13T14:24:45.916 INFO:tasks.rados.rados.0.smithi026.stdout:1689: delete oid 49 current snap is 158 2024-09-13T14:24:45.920 INFO:tasks.rados.rados.0.smithi026.stdout:1689: done (0 left) 2024-09-13T14:24:45.920 INFO:tasks.rados.rados.0.smithi026.stdout:1690: delete oid 40 current snap is 158 2024-09-13T14:24:45.924 INFO:tasks.rados.rados.0.smithi026.stdout:1690: done (0 left) 2024-09-13T14:24:45.924 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 158 2024-09-13T14:24:45.924 INFO:tasks.rados.rados.0.smithi026.stdout:1691: seq_num 551 ranges {1908736=434176,2342912=647168,2990080=450560,3440640=507904} 2024-09-13T14:24:45.936 INFO:tasks.rados.rados.0.smithi026.stdout:1691: writing smithi02683976-15 from 1908736 to 2342912 tid 1 2024-09-13T14:24:45.940 INFO:tasks.rados.rados.0.smithi026.stdout:1691: writing smithi02683976-15 from 2342912 to 2990080 tid 2 2024-09-13T14:24:45.942 INFO:tasks.rados.rados.0.smithi026.stdout:1691: writing smithi02683976-15 from 2990080 to 3440640 tid 3 2024-09-13T14:24:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1691: writing smithi02683976-15 from 3440640 to 3948544 tid 4 2024-09-13T14:24:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1692: delete oid 41 current snap is 158 2024-09-13T14:24:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 1 to smithi02683976-15 2024-09-13T14:24:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 2 to smithi02683976-15 2024-09-13T14:24:45.945 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 3 to smithi02683976-15 2024-09-13T14:24:45.948 INFO:tasks.rados.rados.0.smithi026.stdout:1692: done (1 left) 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1693: read oid 43 snap -1 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1693: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1694: read oid 50 snap -1 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1694: expect deleted 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1695: rollback oid 41 current snap is 158 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 41 to 157 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1696: rollback oid 29 current snap is 158 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 157 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 158 2024-09-13T14:24:45.949 INFO:tasks.rados.rados.0.smithi026.stdout:1697: seq_num 553 ranges {4243456=466944,4710400=417792,5128192=417792,5545984=516096,6062080=253952} 2024-09-13T14:24:45.977 INFO:tasks.rados.rados.0.smithi026.stdout:1697: writing smithi02683976-25 from 4243456 to 4710400 tid 1 2024-09-13T14:24:45.977 INFO:tasks.rados.rados.0.smithi026.stdout:1697: writing smithi02683976-25 from 4710400 to 5128192 tid 2 2024-09-13T14:24:45.979 INFO:tasks.rados.rados.0.smithi026.stdout:1697: writing smithi02683976-25 from 5128192 to 5545984 tid 3 2024-09-13T14:24:45.982 INFO:tasks.rados.rados.0.smithi026.stdout:1697: writing smithi02683976-25 from 5545984 to 6062080 tid 4 2024-09-13T14:24:45.984 INFO:tasks.rados.rados.0.smithi026.stdout:1697: writing smithi02683976-25 from 6062080 to 6316032 tid 5 2024-09-13T14:24:45.984 INFO:tasks.rados.rados.0.smithi026.stdout:1698: snap_create 2024-09-13T14:24:45.984 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 4 to smithi02683976-15 2024-09-13T14:24:45.984 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 5 to smithi02683976-15 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1695: finishing rollback tid 1 to smithi02683976-41 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1695: finishing rollback tid 2 to smithi02683976-41 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 795 (ObjNum 508 snap 140 seq_num 508) dirty exists 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1691: finishing write tid 6 to smithi02683976-15 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 584 (ObjNum 551 snap 158 seq_num 551) dirty exists 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1691: left oid 15 (ObjNum 551 snap 158 seq_num 551) 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1696: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1696: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1696: finishing rollback tid 2 to smithi02683976-29 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 794 (ObjNum 543 snap 156 seq_num 543) dirty exists 2024-09-13T14:24:45.985 INFO:tasks.rados.rados.0.smithi026.stdout:1693: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 1 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 2 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 3 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 4 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 5 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 6 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: finishing write tid 7 to smithi02683976-25 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 625 (ObjNum 553 snap 158 seq_num 553) dirty exists 2024-09-13T14:24:46.132 INFO:tasks.rados.rados.0.smithi026.stdout:1697: left oid 25 (ObjNum 553 snap 158 seq_num 553) 2024-09-13T14:24:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:45 smithi145 ceph-mon[89145]: pgmap v183: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:46.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:45 smithi145 ceph-mon[89145]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:24:46.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:46.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:45 smithi026 ceph-mon[97022]: pgmap v183: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:46.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:45 smithi026 ceph-mon[97022]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:24:46.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:45 smithi026 ceph-mon[99461]: pgmap v183: 121 pgs: 121 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-13T14:24:46.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:45 smithi026 ceph-mon[99461]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:24:46.926 INFO:tasks.rados.rados.0.smithi026.stdout:1691: done (6 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1693: done (5 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1694: done (4 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1695: done (3 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1696: done (2 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1697: done (1 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1698: done (0 left) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1699: rollback oid 13 current snap is 159 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 158 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1700: read oid 46 snap -1 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:1700: expect (ObjNum 542 snap 156 seq_num 542) 2024-09-13T14:24:46.927 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 159 2024-09-13T14:24:46.928 INFO:tasks.rados.rados.0.smithi026.stdout:1701: seq_num 554 ranges {0=483328,483328=704512,1187840=622592,1810432=278528} 2024-09-13T14:24:46.929 INFO:tasks.rados.rados.0.smithi026.stdout:1701: writing smithi02683976-21 from 0 to 483328 tid 1 2024-09-13T14:24:46.934 INFO:tasks.rados.rados.0.smithi026.stdout:1701: writing smithi02683976-21 from 483328 to 1187840 tid 2 2024-09-13T14:24:46.938 INFO:tasks.rados.rados.0.smithi026.stdout:1701: writing smithi02683976-21 from 1187840 to 1810432 tid 3 2024-09-13T14:24:46.940 INFO:tasks.rados.rados.0.smithi026.stdout:1701: writing smithi02683976-21 from 1810432 to 2088960 tid 4 2024-09-13T14:24:46.940 INFO:tasks.rados.rados.0.smithi026.stdout:1702: copy_from oid 42 from oid 44 current snap is 159 2024-09-13T14:24:46.940 INFO:tasks.rados.rados.0.smithi026.stdout:1699: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1699: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1699: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 798 (ObjNum 609 snap 0 seq_num 519533808) dirty exists 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 1 to smithi02683976-21 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1699: done (3 left) 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1703: copy_from oid 43 from oid 5 current snap is 159 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1704: snap_create 2024-09-13T14:24:46.941 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 2 to smithi02683976-21 2024-09-13T14:24:46.945 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 3 to smithi02683976-21 2024-09-13T14:24:46.945 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 4 to smithi02683976-21 2024-09-13T14:24:46.945 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 5 to smithi02683976-21 2024-09-13T14:24:46.946 INFO:tasks.rados.rados.0.smithi026.stdout:1701: finishing write tid 6 to smithi02683976-21 2024-09-13T14:24:46.946 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 882 (ObjNum 554 snap 159 seq_num 554) dirty exists 2024-09-13T14:24:46.946 INFO:tasks.rados.rados.0.smithi026.stdout:1701: left oid 21 (ObjNum 554 snap 159 seq_num 554) 2024-09-13T14:24:46.958 INFO:tasks.rados.rados.0.smithi026.stdout:1700: expect (ObjNum 542 snap 156 seq_num 542) 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:1703: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:1703: finishing copy_from to smithi02683976-43 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 919 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:1702: finishing copy_from racing read to smithi02683976-42 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:1702: finishing copy_from to smithi02683976-42 2024-09-13T14:24:47.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 450 (ObjNum 508 snap 140 seq_num 508) dirty exists 2024-09-13T14:24:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:46 smithi145 ceph-mon[89145]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:24:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:46 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:47.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:47.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:46 smithi026 ceph-mon[97022]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:24:47.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:46 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:46 smithi026 ceph-mon[99461]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:24:47.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:46 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:47.930 INFO:tasks.rados.rados.0.smithi026.stdout:1700: done (4 left) 2024-09-13T14:24:47.930 INFO:tasks.rados.rados.0.smithi026.stdout:1701: done (3 left) 2024-09-13T14:24:47.931 INFO:tasks.rados.rados.0.smithi026.stdout:1702: done (2 left) 2024-09-13T14:24:47.931 INFO:tasks.rados.rados.0.smithi026.stdout:1703: done (1 left) 2024-09-13T14:24:47.931 INFO:tasks.rados.rados.0.smithi026.stdout:1704: done (0 left) 2024-09-13T14:24:47.931 INFO:tasks.rados.rados.0.smithi026.stdout:1705: delete oid 20 current snap is 160 2024-09-13T14:24:47.935 INFO:tasks.rados.rados.0.smithi026.stdout:1705: done (0 left) 2024-09-13T14:24:47.935 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 160 2024-09-13T14:24:47.935 INFO:tasks.rados.rados.0.smithi026.stdout:1706: seq_num 556 ranges {1908736=516096,2424832=524288,2949120=638976,3588096=442368} 2024-09-13T14:24:47.948 INFO:tasks.rados.rados.0.smithi026.stdout:1706: writing smithi02683976-29 from 1908736 to 2424832 tid 1 2024-09-13T14:24:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:1706: writing smithi02683976-29 from 2424832 to 2949120 tid 2 2024-09-13T14:24:47.955 INFO:tasks.rados.rados.0.smithi026.stdout:1706: writing smithi02683976-29 from 2949120 to 3588096 tid 3 2024-09-13T14:24:47.958 INFO:tasks.rados.rados.0.smithi026.stdout:1706: writing smithi02683976-29 from 3588096 to 4030464 tid 4 2024-09-13T14:24:47.958 INFO:tasks.rados.rados.0.smithi026.stdout:1707: rmattr oid 24 current snap is 160 2024-09-13T14:24:47.958 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 1 to smithi02683976-29 2024-09-13T14:24:47.959 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 2 to smithi02683976-29 2024-09-13T14:24:47.959 INFO:tasks.rados.rados.0.smithi026.stdout:1708: snap_remove snap 159 2024-09-13T14:24:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:47 smithi145 ceph-mon[89145]: pgmap v186: 121 pgs: 121 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-09-13T14:24:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:47 smithi145 ceph-mon[89145]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:24:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:47 smithi026 ceph-mon[99461]: pgmap v186: 121 pgs: 121 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-09-13T14:24:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:47 smithi026 ceph-mon[99461]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:24:48.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:47 smithi026 ceph-mon[97022]: pgmap v186: 121 pgs: 121 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-09-13T14:24:48.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:47 smithi026 ceph-mon[97022]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:24:48.912 INFO:tasks.rados.rados.0.smithi026.stdout:1708: done (2 left) 2024-09-13T14:24:48.912 INFO:tasks.rados.rados.0.smithi026.stdout:1709: delete oid 11 current snap is 160 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 3 to smithi02683976-29 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 4 to smithi02683976-29 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 626 (ObjNum 544 snap 156 seq_num 544) dirty exists 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 5 to smithi02683976-29 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1706: finishing write tid 6 to smithi02683976-29 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 925 (ObjNum 556 snap 160 seq_num 556) dirty exists 2024-09-13T14:24:48.913 INFO:tasks.rados.rados.0.smithi026.stdout:1706: left oid 29 (ObjNum 556 snap 160 seq_num 556) 2024-09-13T14:24:48.917 INFO:tasks.rados.rados.0.smithi026.stdout:1706: done (2 left) 2024-09-13T14:24:48.918 INFO:tasks.rados.rados.0.smithi026.stdout:1707: done (1 left) 2024-09-13T14:24:48.918 INFO:tasks.rados.rados.0.smithi026.stdout:1709: done (0 left) 2024-09-13T14:24:48.918 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 160 2024-09-13T14:24:48.918 INFO:tasks.rados.rados.0.smithi026.stdout:1710: seq_num 558 ranges {0=548864,548864=688128,1236992=606208,1843200=311296} 2024-09-13T14:24:48.921 INFO:tasks.rados.rados.0.smithi026.stdout:1710: writing smithi02683976-50 from 0 to 548864 tid 1 2024-09-13T14:24:48.926 INFO:tasks.rados.rados.0.smithi026.stdout:1710: writing smithi02683976-50 from 548864 to 1236992 tid 2 2024-09-13T14:24:48.929 INFO:tasks.rados.rados.0.smithi026.stdout:1710: writing smithi02683976-50 from 1236992 to 1843200 tid 3 2024-09-13T14:24:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:1710: writing smithi02683976-50 from 1843200 to 2154496 tid 4 2024-09-13T14:24:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 1 to smithi02683976-50 2024-09-13T14:24:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 2 to smithi02683976-50 2024-09-13T14:24:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:1711: snap_remove snap 158 2024-09-13T14:24:49.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[99461]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:24:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:49.194 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[99461]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:24:49.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[97022]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:24:49.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:49.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:48 smithi026 ceph-mon[97022]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:24:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:48 smithi145 ceph-mon[89145]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:24:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:48 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:24:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:48 smithi145 ceph-mon[89145]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:24:49.934 INFO:tasks.rados.rados.0.smithi026.stdout:1711: done (1 left) 2024-09-13T14:24:49.934 INFO:tasks.rados.rados.0.smithi026.stdout:1712: read oid 37 snap -1 2024-09-13T14:24:49.935 INFO:tasks.rados.rados.0.smithi026.stdout:1712: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:49.935 INFO:tasks.rados.rados.0.smithi026.stdout:1713: rmattr oid 9 current snap is 160 2024-09-13T14:24:49.935 INFO:tasks.rados.rados.0.smithi026.stdout:1713: done (2 left) 2024-09-13T14:24:49.935 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 160 2024-09-13T14:24:49.935 INFO:tasks.rados.rados.0.smithi026.stdout:1714: seq_num 559 ranges {8306688=565248,8871936=573440,9445376=409600,9854976=622592} 2024-09-13T14:24:49.984 INFO:tasks.rados.rados.0.smithi026.stdout:1714: writing smithi02683976-44 from 8306688 to 8871936 tid 1 2024-09-13T14:24:49.987 INFO:tasks.rados.rados.0.smithi026.stdout:1714: writing smithi02683976-44 from 8871936 to 9445376 tid 2 2024-09-13T14:24:49.990 INFO:tasks.rados.rados.0.smithi026.stdout:1714: writing smithi02683976-44 from 9445376 to 9854976 tid 3 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1714: writing smithi02683976-44 from 9854976 to 10477568 tid 4 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1715: delete oid 8 current snap is 160 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 3 to smithi02683976-50 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 4 to smithi02683976-50 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 5 to smithi02683976-50 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1710: finishing write tid 6 to smithi02683976-50 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 889 (ObjNum 558 snap 160 seq_num 558) dirty exists 2024-09-13T14:24:49.994 INFO:tasks.rados.rados.0.smithi026.stdout:1710: left oid 50 (ObjNum 558 snap 160 seq_num 558) 2024-09-13T14:24:49.995 INFO:tasks.rados.rados.0.smithi026.stdout:1712: expect (ObjNum 441 snap 122 seq_num 441) 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[99461]: Upgrade: osd.2 is safe to restart 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[99461]: pgmap v189: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[97022]: Upgrade: osd.2 is safe to restart 2024-09-13T14:24:50.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:49 smithi026 ceph-mon[97022]: pgmap v189: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-13T14:24:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:49 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:24:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:49 smithi145 ceph-mon[89145]: Upgrade: osd.2 is safe to restart 2024-09-13T14:24:50.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:49 smithi145 ceph-mon[89145]: pgmap v189: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 1 to smithi02683976-44 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 2 to smithi02683976-44 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 3 to smithi02683976-44 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 4 to smithi02683976-44 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 5 to smithi02683976-44 2024-09-13T14:24:50.641 INFO:tasks.rados.rados.0.smithi026.stdout:1714: finishing write tid 6 to smithi02683976-44 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 460 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:1714: left oid 44 (ObjNum 559 snap 160 seq_num 559) 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:1710: done (3 left) 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:1712: done (2 left) 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:1714: done (1 left) 2024-09-13T14:24:50.642 INFO:tasks.rados.rados.0.smithi026.stdout:1715: done (0 left) 2024-09-13T14:24:50.643 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 160 2024-09-13T14:24:50.643 INFO:tasks.rados.rados.0.smithi026.stdout:1716: seq_num 561 ranges {0=598016,598016=745472,1343488=778240,2121728=81920} 2024-09-13T14:24:50.643 INFO:tasks.rados.rados.0.smithi026.stdout:1716: writing smithi02683976-6 from 0 to 598016 tid 1 2024-09-13T14:24:50.647 INFO:tasks.rados.rados.0.smithi026.stdout:1716: writing smithi02683976-6 from 598016 to 1343488 tid 2 2024-09-13T14:24:50.652 INFO:tasks.rados.rados.0.smithi026.stdout:1716: writing smithi02683976-6 from 1343488 to 2121728 tid 3 2024-09-13T14:24:50.653 INFO:tasks.rados.rados.0.smithi026.stdout:1716: writing smithi02683976-6 from 2121728 to 2203648 tid 4 2024-09-13T14:24:50.653 INFO:tasks.rados.rados.0.smithi026.stdout:1717: snap_create 2024-09-13T14:24:50.653 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 1 to smithi02683976-6 2024-09-13T14:24:50.653 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 2 to smithi02683976-6 2024-09-13T14:24:50.656 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 3 to smithi02683976-6 2024-09-13T14:24:50.656 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 4 to smithi02683976-6 2024-09-13T14:24:50.656 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 5 to smithi02683976-6 2024-09-13T14:24:50.657 INFO:tasks.rados.rados.0.smithi026.stdout:1716: finishing write tid 6 to smithi02683976-6 2024-09-13T14:24:50.657 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 896 (ObjNum 561 snap 160 seq_num 561) dirty exists 2024-09-13T14:24:50.658 INFO:tasks.rados.rados.0.smithi026.stdout:1716: left oid 6 (ObjNum 561 snap 160 seq_num 561) 2024-09-13T14:24:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:51.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: Upgrade: Updating osd.2 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[99461]: Deploying daemon osd.2 on smithi026 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: Upgrade: Updating osd.2 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:50 smithi026 ceph-mon[97022]: Deploying daemon osd.2 on smithi026 2024-09-13T14:24:51.193 INFO:tasks.rados.rados.0.smithi026.stdout:1716: done (1 left) 2024-09-13T14:24:51.193 INFO:tasks.rados.rados.0.smithi026.stdout:1717: done (0 left) 2024-09-13T14:24:51.193 INFO:tasks.rados.rados.0.smithi026.stdout:1718: rollback oid 21 current snap is 161 2024-09-13T14:24:51.193 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 160 2024-09-13T14:24:51.193 INFO:tasks.rados.rados.0.smithi026.stdout:1719: read oid 13 snap 160 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1719: expect (ObjNum 518854880 snap 21981 seq_num 519780832) 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1720: rollback oid 28 current snap is 161 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 160 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1721: read oid 19 snap -1 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1721: expect (ObjNum 531 snap 148 seq_num 531) 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1722: rollback oid 25 current snap is 161 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 157 2024-09-13T14:24:51.194 INFO:tasks.rados.rados.0.smithi026.stdout:1723: rollback oid 29 current snap is 161 2024-09-13T14:24:51.195 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 157 2024-09-13T14:24:51.195 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 161 2024-09-13T14:24:51.195 INFO:tasks.rados.rados.0.smithi026.stdout:1724: seq_num 562 ranges {1769472=622592,2392064=630784,3022848=581632,3604480=393216} 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: Upgrade: Updating osd.2 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:50 smithi145 ceph-mon[89145]: Deploying daemon osd.2 on smithi026 2024-09-13T14:24:51.206 INFO:tasks.rados.rados.0.smithi026.stdout:1724: writing smithi02683976-7 from 1769472 to 2392064 tid 1 2024-09-13T14:24:51.210 INFO:tasks.rados.rados.0.smithi026.stdout:1724: writing smithi02683976-7 from 2392064 to 3022848 tid 2 2024-09-13T14:24:51.214 INFO:tasks.rados.rados.0.smithi026.stdout:1724: writing smithi02683976-7 from 3022848 to 3604480 tid 3 2024-09-13T14:24:51.217 INFO:tasks.rados.rados.0.smithi026.stdout:1724: writing smithi02683976-7 from 3604480 to 3997696 tid 4 2024-09-13T14:24:51.217 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 161 2024-09-13T14:24:51.217 INFO:tasks.rados.rados.0.smithi026.stdout:1725: seq_num 563 ranges {0=638976,638976=565248,1204224=794624,1998848=245760} 2024-09-13T14:24:51.220 INFO:tasks.rados.rados.0.smithi026.stdout:1725: writing smithi02683976-11 from 0 to 638976 tid 1 2024-09-13T14:24:51.223 INFO:tasks.rados.rados.0.smithi026.stdout:1725: writing smithi02683976-11 from 638976 to 1204224 tid 2 2024-09-13T14:24:51.228 INFO:tasks.rados.rados.0.smithi026.stdout:1725: writing smithi02683976-11 from 1204224 to 1998848 tid 3 2024-09-13T14:24:51.229 INFO:tasks.rados.rados.0.smithi026.stdout:1725: writing smithi02683976-11 from 1998848 to 2244608 tid 4 2024-09-13T14:24:51.229 INFO:tasks.rados.rados.0.smithi026.stdout:1726: copy_from oid 38 from oid 26 current snap is 161 2024-09-13T14:24:51.229 INFO:tasks.rados.rados.0.smithi026.stdout:1719: expect (ObjNum 518854880 snap 21981 seq_num 519780832) 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1722: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1722: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1722: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 629 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1720: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1720: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1720: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 808 (ObjNum 530 snap 148 seq_num 530) dirty exists 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1718: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:24:51.233 INFO:tasks.rados.rados.0.smithi026.stdout:1718: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1718: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 900 (ObjNum 554 snap 159 seq_num 554) dirty exists 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1718: done (8 left) 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1719: done (7 left) 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1720: done (6 left) 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1722: done (5 left) 2024-09-13T14:24:51.234 INFO:tasks.rados.rados.0.smithi026.stdout:1727: snap_remove snap 160 2024-09-13T14:24:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:24:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:24:51] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1727: done (5 left) 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1728: read oid 35 snap 157 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1728: expect deleted 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1729: rmattr oid 28 current snap is 161 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1729: done (6 left) 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 161 2024-09-13T14:24:52.197 INFO:tasks.rados.rados.0.smithi026.stdout:1723: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:24:52.198 INFO:tasks.rados.rados.0.smithi026.stdout:1723: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:24:52.198 INFO:tasks.rados.rados.0.smithi026.stdout:1723: finishing rollback tid 2 to smithi02683976-29 2024-09-13T14:24:52.198 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 928 (ObjNum 543 snap 156 seq_num 543) dirty exists 2024-09-13T14:24:52.198 INFO:tasks.rados.rados.0.smithi026.stdout:1730: seq_num 564 ranges {0=655360,655360=442368,1097728=598016,1695744=565248} 2024-09-13T14:24:52.201 INFO:tasks.rados.rados.0.smithi026.stdout:1730: writing smithi02683976-45 from 0 to 655360 tid 1 2024-09-13T14:24:52.204 INFO:tasks.rados.rados.0.smithi026.stdout:1730: writing smithi02683976-45 from 655360 to 1097728 tid 2 2024-09-13T14:24:52.209 INFO:tasks.rados.rados.0.smithi026.stdout:1730: writing smithi02683976-45 from 1097728 to 1695744 tid 3 2024-09-13T14:24:52.213 INFO:tasks.rados.rados.0.smithi026.stdout:1730: writing smithi02683976-45 from 1695744 to 2260992 tid 4 2024-09-13T14:24:52.213 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 1 to smithi02683976-7 2024-09-13T14:24:52.213 INFO:tasks.rados.rados.0.smithi026.stdout:1721: expect (ObjNum 531 snap 148 seq_num 531) 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 2 to smithi02683976-7 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 3 to smithi02683976-7 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1721: done (6 left) 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1723: done (5 left) 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1731: setattr oid 3 current snap is 161 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 4 to smithi02683976-7 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 5 to smithi02683976-7 2024-09-13T14:24:52.295 INFO:tasks.rados.rados.0.smithi026.stdout:1724: finishing write tid 6 to smithi02683976-7 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 523 (ObjNum 562 snap 161 seq_num 562) dirty exists 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1724: left oid 7 (ObjNum 562 snap 161 seq_num 562) 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 1 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 2 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 3 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 4 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 5 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: finishing write tid 6 to smithi02683976-11 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 905 (ObjNum 563 snap 161 seq_num 563) dirty exists 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1725: left oid 11 (ObjNum 563 snap 161 seq_num 563) 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1726: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:1726: finishing copy_from to smithi02683976-38 2024-09-13T14:24:52.296 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 768 (ObjNum 129 snap 0 seq_num 519530160) dirty exists 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 1 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 2 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 3 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 4 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 5 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: finishing write tid 6 to smithi02683976-45 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 297 (ObjNum 564 snap 161 seq_num 564) dirty exists 2024-09-13T14:24:52.297 INFO:tasks.rados.rados.0.smithi026.stdout:1730: left oid 45 (ObjNum 564 snap 161 seq_num 564) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1724: done (5 left) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1725: done (4 left) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1726: done (3 left) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1728: done (2 left) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1730: done (1 left) 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 161 2024-09-13T14:24:52.298 INFO:tasks.rados.rados.0.smithi026.stdout:1732: seq_num 565 ranges {0=671744,671744=729088,1400832=753664,2154496=122880} 2024-09-13T14:24:52.300 INFO:tasks.rados.rados.0.smithi026.stdout:1732: writing smithi02683976-20 from 0 to 671744 tid 1 2024-09-13T14:24:52.304 INFO:tasks.rados.rados.0.smithi026.stdout:1732: writing smithi02683976-20 from 671744 to 1400832 tid 2 2024-09-13T14:24:52.309 INFO:tasks.rados.rados.0.smithi026.stdout:1732: writing smithi02683976-20 from 1400832 to 2154496 tid 3 2024-09-13T14:24:52.312 INFO:tasks.rados.rados.0.smithi026.stdout:1732: writing smithi02683976-20 from 2154496 to 2277376 tid 4 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1733: copy_from oid 45 from oid 28 current snap is 161 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 927 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 1 to smithi02683976-20 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 2 to smithi02683976-20 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1731: done (2 left) 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1734: rollback oid 3 current snap is 161 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 157 2024-09-13T14:24:52.313 INFO:tasks.rados.rados.0.smithi026.stdout:1735: delete oid 29 current snap is 161 2024-09-13T14:24:52.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:52 smithi026 ceph-mon[99461]: pgmap v192: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 976 KiB/s wr, 2 op/s 2024-09-13T14:24:52.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:52 smithi026 ceph-mon[99461]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:24:52.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:52 smithi026 ceph-mon[97022]: pgmap v192: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 976 KiB/s wr, 2 op/s 2024-09-13T14:24:52.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:52 smithi026 ceph-mon[97022]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:24:52.317 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 3 to smithi02683976-20 2024-09-13T14:24:52.317 INFO:tasks.rados.rados.0.smithi026.stdout:1734: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:24:52.318 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 4 to smithi02683976-20 2024-09-13T14:24:52.318 INFO:tasks.rados.rados.0.smithi026.stdout:1734: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:24:52.318 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 5 to smithi02683976-20 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1734: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 930 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1734: done (3 left) 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1735: done (2 left) 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1736: copy_from oid 22 from oid 3 current snap is 161 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1737: copy_from oid 24 from oid 42 current snap is 161 2024-09-13T14:24:52.319 INFO:tasks.rados.rados.0.smithi026.stdout:1738: read oid 2 snap -1 2024-09-13T14:24:52.320 INFO:tasks.rados.rados.0.smithi026.stdout:1738: expect (ObjNum 534 snap 149 seq_num 534) 2024-09-13T14:24:52.320 INFO:tasks.rados.rados.0.smithi026.stdout:1739: snap_create 2024-09-13T14:24:52.321 INFO:tasks.rados.rados.0.smithi026.stdout:1732: finishing write tid 6 to smithi02683976-20 2024-09-13T14:24:52.321 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 933 (ObjNum 565 snap 161 seq_num 565) dirty exists 2024-09-13T14:24:52.321 INFO:tasks.rados.rados.0.smithi026.stdout:1732: left oid 20 (ObjNum 565 snap 161 seq_num 565) 2024-09-13T14:24:52.331 INFO:tasks.rados.rados.0.smithi026.stdout:1733: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:24:52.333 INFO:tasks.rados.rados.0.smithi026.stdout:1733: finishing copy_from to smithi02683976-45 2024-09-13T14:24:52.333 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 808 (ObjNum 530 snap 148 seq_num 530) dirty exists 2024-09-13T14:24:52.340 INFO:tasks.rados.rados.0.smithi026.stdout:1738: expect (ObjNum 534 snap 149 seq_num 534) 2024-09-13T14:24:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:52 smithi145 ceph-mon[89145]: pgmap v192: 121 pgs: 121 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 976 KiB/s wr, 2 op/s 2024-09-13T14:24:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:52 smithi145 ceph-mon[89145]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:24:52.580 INFO:tasks.rados.rados.0.smithi026.stdout:1736: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:24:52.580 INFO:tasks.rados.rados.0.smithi026.stdout:1736: finishing copy_from to smithi02683976-22 2024-09-13T14:24:52.580 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 930 (ObjNum 216 snap 53 seq_num 216) dirty exists 2024-09-13T14:24:52.581 INFO:tasks.rados.rados.0.smithi026.stdout:1737: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:24:52.581 INFO:tasks.rados.rados.0.smithi026.stdout:1737: finishing copy_from to smithi02683976-24 2024-09-13T14:24:52.581 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 450 (ObjNum 508 snap 140 seq_num 508) dirty exists 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1732: done (5 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1733: done (4 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1736: done (3 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1737: done (2 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1738: done (1 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1739: done (0 left) 2024-09-13T14:24:53.202 INFO:tasks.rados.rados.0.smithi026.stdout:1740: snap_create 2024-09-13T14:24:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:53 smithi145 ceph-mon[89145]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:24:53.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:53 smithi026 ceph-mon[97022]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:24:53.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:53 smithi026 ceph-mon[99461]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:24:53.556 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:53 smithi026 systemd[1]: Stopping Ceph osd.2 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:24:53.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:53 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[55591]: 2024-09-13T14:24:53.554+0000 7fd1cfc07640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:24:53.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:53 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[55591]: 2024-09-13T14:24:53.554+0000 7fd1cfc07640 -1 osd.2 398 *** Got signal Terminated *** 2024-09-13T14:24:53.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:53 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[55591]: 2024-09-13T14:24:53.554+0000 7fd1cfc07640 -1 osd.2 398 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:24:53.916 INFO:tasks.rados.rados.0.smithi026.stdout:1740: done (0 left) 2024-09-13T14:24:53.916 INFO:tasks.rados.rados.0.smithi026.stdout:1741: snap_remove snap 157 2024-09-13T14:24:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-mon[89145]: pgmap v195: 121 pgs: 121 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-mon[89145]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:24:54.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-mon[89145]: osd.2 marked itself down and dead 2024-09-13T14:24:54.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:54.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-mon[89145]: osdmap e400: 8 total, 7 up, 8 in 2024-09-13T14:24:54.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:54.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[99461]: pgmap v195: 121 pgs: 121 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:54.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[99461]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:24:54.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[99461]: osd.2 marked itself down and dead 2024-09-13T14:24:54.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:54.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[99461]: osdmap e400: 8 total, 7 up, 8 in 2024-09-13T14:24:54.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[97022]: pgmap v195: 121 pgs: 121 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:54.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[97022]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:24:54.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[97022]: osd.2 marked itself down and dead 2024-09-13T14:24:54.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:24:54.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:54 smithi026 ceph-mon[97022]: osdmap e400: 8 total, 7 up, 8 in 2024-09-13T14:24:54.854 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:54 smithi026 podman[114866]: 2024-09-13 14:24:54.464893224 +0000 UTC m=+1.013802139 container died b7165e08f421c75ebb7af796a2a81248ac47bb2b4c5255d524d1715dcde2a4df (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:24:54.923 INFO:tasks.rados.rados.0.smithi026.stdout:1741: done (0 left) 2024-09-13T14:24:54.923 INFO:tasks.rados.rados.0.smithi026.stdout:1742: rollback oid 35 current snap is 163 2024-09-13T14:24:54.923 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 35 to 162 2024-09-13T14:24:54.924 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 163 2024-09-13T14:24:54.924 INFO:tasks.rados.rados.0.smithi026.stdout:1743: seq_num 567 ranges {0=704512,704512=499712,1204224=770048,1974272=335872} 2024-09-13T14:24:54.928 INFO:tasks.rados.rados.0.smithi026.stdout:1743: writing smithi02683976-29 from 0 to 704512 tid 1 2024-09-13T14:24:54.932 INFO:tasks.rados.rados.0.smithi026.stdout:1743: writing smithi02683976-29 from 704512 to 1204224 tid 2 2024-09-13T14:24:54.937 INFO:tasks.rados.rados.0.smithi026.stdout:1743: writing smithi02683976-29 from 1204224 to 1974272 tid 3 2024-09-13T14:24:54.939 INFO:tasks.rados.rados.0.smithi026.stdout:1743: writing smithi02683976-29 from 1974272 to 2310144 tid 4 2024-09-13T14:24:54.940 INFO:tasks.rados.rados.0.smithi026.stdout:1744: delete oid 30 current snap is 163 2024-09-13T14:24:54.940 INFO:tasks.rados.rados.0.smithi026.stdout:1742: finishing rollback tid 1 to smithi02683976-35 2024-09-13T14:24:54.940 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 0 (ObjNum 1265 snap 0 seq_num 3913774344) dirty dne 2024-09-13T14:24:54.940 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 1 to smithi02683976-29 2024-09-13T14:24:54.941 INFO:tasks.rados.rados.0.smithi026.stdout:1742: done (2 left) 2024-09-13T14:24:54.941 INFO:tasks.rados.rados.0.smithi026.stdout:1744: done (1 left) 2024-09-13T14:24:54.941 INFO:tasks.rados.rados.0.smithi026.stdout:1745: delete oid 19 current snap is 163 2024-09-13T14:24:54.941 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 2 to smithi02683976-29 2024-09-13T14:24:54.945 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 3 to smithi02683976-29 2024-09-13T14:24:54.945 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 4 to smithi02683976-29 2024-09-13T14:24:54.948 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 5 to smithi02683976-29 2024-09-13T14:24:54.949 INFO:tasks.rados.rados.0.smithi026.stdout:1745: done (1 left) 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:1746: read oid 28 snap -1 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:1746: expect (ObjNum 530 snap 148 seq_num 530) 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:1747: rollback oid 32 current snap is 163 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 161 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:1748: read oid 34 snap -1 2024-09-13T14:24:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:1748: expect deleted 2024-09-13T14:24:54.951 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 163 2024-09-13T14:24:54.951 INFO:tasks.rados.rados.0.smithi026.stdout:1749: seq_num 570 ranges {2277376=753664,3031040=548864,3579904=540672,4120576=516096} 2024-09-13T14:24:54.966 INFO:tasks.rados.rados.0.smithi026.stdout:1749: writing smithi02683976-20 from 2277376 to 3031040 tid 1 2024-09-13T14:24:54.969 INFO:tasks.rados.rados.0.smithi026.stdout:1749: writing smithi02683976-20 from 3031040 to 3579904 tid 2 2024-09-13T14:24:54.973 INFO:tasks.rados.rados.0.smithi026.stdout:1749: writing smithi02683976-20 from 3579904 to 4120576 tid 3 2024-09-13T14:24:54.976 INFO:tasks.rados.rados.0.smithi026.stdout:1749: writing smithi02683976-20 from 4120576 to 4636672 tid 4 2024-09-13T14:24:54.977 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 163 2024-09-13T14:24:54.977 INFO:tasks.rados.rados.0.smithi026.stdout:1750: seq_num 571 ranges {3981312=770048,4751360=483328,5234688=753664,5988352=368640} 2024-09-13T14:24:55.003 INFO:tasks.rados.rados.0.smithi026.stdout:1750: writing smithi02683976-45 from 3981312 to 4751360 tid 1 2024-09-13T14:24:55.005 INFO:tasks.rados.rados.0.smithi026.stdout:1750: writing smithi02683976-45 from 4751360 to 5234688 tid 2 2024-09-13T14:24:55.010 INFO:tasks.rados.rados.0.smithi026.stdout:1750: writing smithi02683976-45 from 5234688 to 5988352 tid 3 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:1750: writing smithi02683976-45 from 5988352 to 6356992 tid 4 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 163 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:1743: finishing write tid 6 to smithi02683976-29 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 939 (ObjNum 567 snap 163 seq_num 567) dirty exists 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:1743: left oid 29 (ObjNum 567 snap 163 seq_num 567) 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:1747: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:24:55.013 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 0 (ObjNum 225 snap 0 seq_num 519592256) dirty dne 2024-09-13T14:24:55.014 INFO:tasks.rados.rados.0.smithi026.stdout:1746: expect (ObjNum 530 snap 148 seq_num 530) 2024-09-13T14:24:55.165 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:54 smithi026 podman[114866]: 2024-09-13 14:24:54.928154496 +0000 UTC m=+1.477063405 container cleanup b7165e08f421c75ebb7af796a2a81248ac47bb2b4c5255d524d1715dcde2a4df (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:24:55.165 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:54 smithi026 bash[114866]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:24:55.171 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 1 to smithi02683976-20 2024-09-13T14:24:55.171 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 2 to smithi02683976-20 2024-09-13T14:24:55.171 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 3 to smithi02683976-20 2024-09-13T14:24:55.171 INFO:tasks.rados.rados.0.smithi026.stdout:1751: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2024-09-13T14:24:55.175 INFO:tasks.rados.rados.0.smithi026.stdout:1751: writing smithi02683976-9 from 0 to 786432 tid 1 2024-09-13T14:24:55.179 INFO:tasks.rados.rados.0.smithi026.stdout:1751: writing smithi02683976-9 from 786432 to 1556480 tid 2 2024-09-13T14:24:55.182 INFO:tasks.rados.rados.0.smithi026.stdout:1751: writing smithi02683976-9 from 1556480 to 2113536 tid 3 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1751: writing smithi02683976-9 from 2113536 to 2392064 tid 4 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1743: done (6 left) 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1746: done (5 left) 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1747: done (4 left) 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1748: done (3 left) 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 163 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 4 to smithi02683976-20 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 5 to smithi02683976-20 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1749: finishing write tid 6 to smithi02683976-20 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 945 (ObjNum 570 snap 163 seq_num 570) dirty exists 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1749: left oid 20 (ObjNum 570 snap 163 seq_num 570) 2024-09-13T14:24:55.185 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 1 to smithi02683976-45 2024-09-13T14:24:55.186 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 2 to smithi02683976-45 2024-09-13T14:24:55.186 INFO:tasks.rados.rados.0.smithi026.stdout:1752: seq_num 573 ranges {2088960=802816,2891776=655360,3547136=720896,4268032=229376} 2024-09-13T14:24:55.200 INFO:tasks.rados.rados.0.smithi026.stdout:1752: writing smithi02683976-21 from 2088960 to 2891776 tid 1 2024-09-13T14:24:55.204 INFO:tasks.rados.rados.0.smithi026.stdout:1752: writing smithi02683976-21 from 2891776 to 3547136 tid 2 2024-09-13T14:24:55.209 INFO:tasks.rados.rados.0.smithi026.stdout:1752: writing smithi02683976-21 from 3547136 to 4268032 tid 3 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1752: writing smithi02683976-21 from 4268032 to 4497408 tid 4 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1749: done (3 left) 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 163 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 3 to smithi02683976-45 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 4 to smithi02683976-45 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 5 to smithi02683976-45 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:1750: finishing write tid 6 to smithi02683976-45 2024-09-13T14:24:55.211 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 813 (ObjNum 571 snap 163 seq_num 571) dirty exists 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1750: left oid 45 (ObjNum 571 snap 163 seq_num 571) 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1753: seq_num 574 ranges {0=16384} 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1753: writing smithi02683976-34 from 0 to 16384 tid 1 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1750: done (3 left) 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1754: delete oid 4 current snap is 163 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 1 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 2 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 3 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 4 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 5 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:1751: finishing write tid 6 to smithi02683976-9 2024-09-13T14:24:55.212 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 914 (ObjNum 572 snap 163 seq_num 572) dirty exists 2024-09-13T14:24:55.213 INFO:tasks.rados.rados.0.smithi026.stdout:1751: left oid 9 (ObjNum 572 snap 163 seq_num 572) 2024-09-13T14:24:55.213 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 1 to smithi02683976-21 2024-09-13T14:24:55.213 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 2 to smithi02683976-21 2024-09-13T14:24:55.213 INFO:tasks.rados.rados.0.smithi026.stdout:1751: done (3 left) 2024-09-13T14:24:55.213 INFO:tasks.rados.rados.0.smithi026.stdout:1754: done (2 left) 2024-09-13T14:24:55.214 INFO:tasks.rados.rados.0.smithi026.stdout:1755: rollback oid 47 current snap is 163 2024-09-13T14:24:55.214 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 162 2024-09-13T14:24:55.214 INFO:tasks.rados.rados.0.smithi026.stdout:1756: snap_remove snap 161 2024-09-13T14:24:55.513 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114887]: 2024-09-13 14:24:55.170741319 +0000 UTC m=+0.697920429 container remove b7165e08f421c75ebb7af796a2a81248ac47bb2b4c5255d524d1715dcde2a4df (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:24:55.799 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.416194469 +0000 UTC m=+0.016635383 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:55.800 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.563597953 +0000 UTC m=+0.164038866 container create 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:24:55.800 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.799420428 +0000 UTC m=+0.399861342 container init 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:24:56.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:55 smithi026 ceph-mon[99461]: pgmap v198: 121 pgs: 14 stale+active+clean, 107 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:56.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:55 smithi026 ceph-mon[99461]: osdmap e401: 8 total, 7 up, 8 in 2024-09-13T14:24:56.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:55 smithi026 ceph-mon[97022]: pgmap v198: 121 pgs: 14 stale+active+clean, 107 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:56.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:55 smithi026 ceph-mon[97022]: osdmap e401: 8 total, 7 up, 8 in 2024-09-13T14:24:56.166 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.804439366 +0000 UTC m=+0.404880282 container start 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:24:56.166 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.882598107 +0000 UTC m=+0.483039020 container attach 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T14:24:56.166 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:55 smithi026 podman[114982]: 2024-09-13 14:24:55.998108056 +0000 UTC m=+0.598548974 container died 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD) 2024-09-13T14:24:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:55 smithi145 ceph-mon[89145]: pgmap v198: 121 pgs: 14 stale+active+clean, 107 active+clean; 182 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.7 MiB/s wr, 18 op/s 2024-09-13T14:24:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:55 smithi145 ceph-mon[89145]: osdmap e401: 8 total, 7 up, 8 in 2024-09-13T14:24:56.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:56.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1756: done (3 left) 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1757: rmattr oid 33 current snap is 163 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 3 to smithi02683976-21 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 4 to smithi02683976-21 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 5 to smithi02683976-21 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1753: finishing write tid 1 to smithi02683976-34 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1753: finishing write tid 2 to smithi02683976-34 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:1752: finishing write tid 6 to smithi02683976-21 2024-09-13T14:24:56.207 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 920 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1752: left oid 21 (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1753: finishing write tid 3 to smithi02683976-34 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 797 (ObjNum 574 snap 163 seq_num 574) dirty exists 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1753: left oid 34 (ObjNum 574 snap 163 seq_num 574) 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1755: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1755: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1755: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 720 (ObjNum 548 snap 157 seq_num 548) dirty exists 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1752: done (3 left) 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1753: done (2 left) 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1755: done (1 left) 2024-09-13T14:24:56.208 INFO:tasks.rados.rados.0.smithi026.stdout:1758: snap_create 2024-09-13T14:24:56.211 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 933 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:24:56.915 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:56 smithi026 podman[115025]: 2024-09-13 14:24:56.594726798 +0000 UTC m=+0.586188823 container remove 6e5a1866962253b2e52cee2473c64c911c501a6f12def04c7f07bd151f51f89c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:24:56.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:56 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T14:24:56.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:56 smithi026 systemd[1]: Stopped Ceph osd.2 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:24:56.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:56 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.2.service: Consumed 16.676s CPU time. 2024-09-13T14:24:56.916 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:56 smithi026 systemd[1]: Starting Ceph osd.2 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:24:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:57.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:57 smithi145 ceph-mon[89145]: osdmap e402: 8 total, 7 up, 8 in 2024-09-13T14:24:57.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:24:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:24:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1757: done (1 left) 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1758: done (0 left) 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 164 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1759: seq_num 576 ranges {0=49152} 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1759: writing smithi02683976-32 from 0 to 49152 tid 1 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1760: rollback oid 4 current snap is 164 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 162 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 164 2024-09-13T14:24:57.210 INFO:tasks.rados.rados.0.smithi026.stdout:1761: seq_num 577 ranges {5881856=65536} 2024-09-13T14:24:57.243 INFO:tasks.rados.rados.0.smithi026.stdout:1761: writing smithi02683976-17 from 5881856 to 5947392 tid 1 2024-09-13T14:24:57.243 INFO:tasks.rados.rados.0.smithi026.stdout:1762: rollback oid 34 current snap is 164 2024-09-13T14:24:57.243 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 163 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1759: finishing write tid 1 to smithi02683976-32 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1759: finishing write tid 2 to smithi02683976-32 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1760: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1760: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 587 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1759: finishing write tid 3 to smithi02683976-32 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 797 (ObjNum 576 snap 164 seq_num 576) dirty exists 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1759: left oid 32 (ObjNum 576 snap 164 seq_num 576) 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1759: done (3 left) 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1760: done (2 left) 2024-09-13T14:24:57.244 INFO:tasks.rados.rados.0.smithi026.stdout:1763: setattr oid 16 current snap is 164 2024-09-13T14:24:57.245 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 164 2024-09-13T14:24:57.245 INFO:tasks.rados.rados.0.smithi026.stdout:1764: seq_num 578 ranges {8306688=81920} 2024-09-13T14:24:57.290 INFO:tasks.rados.rados.0.smithi026.stdout:1764: writing smithi02683976-41 from 8306688 to 8388608 tid 1 2024-09-13T14:24:57.290 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 164 2024-09-13T14:24:57.290 INFO:tasks.rados.rados.0.smithi026.stdout:1765: seq_num 579 ranges {5029888=98304} 2024-09-13T14:24:57.319 INFO:tasks.rados.rados.0.smithi026.stdout:1765: writing smithi02683976-12 from 5029888 to 5128192 tid 1 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1766: read oid 5 snap -1 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1766: expect (ObjNum 534 snap 149 seq_num 534) 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1767: copy_from oid 29 from oid 13 current snap is 164 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 946 (ObjNum 518 snap 142 seq_num 518) dirty exists 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1761: finishing write tid 1 to smithi02683976-17 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1761: finishing write tid 2 to smithi02683976-17 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1763: done (6 left) 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 164 2024-09-13T14:24:57.320 INFO:tasks.rados.rados.0.smithi026.stdout:1762: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:24:57.321 INFO:tasks.rados.rados.0.smithi026.stdout:1762: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:24:57.321 INFO:tasks.rados.rados.0.smithi026.stdout:1768: seq_num 580 ranges {8306688=114688} 2024-09-13T14:24:57.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:57.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:57.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[99461]: osdmap e402: 8 total, 7 up, 8 in 2024-09-13T14:24:57.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:57.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:57.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-mon[97022]: osdmap e402: 8 total, 7 up, 8 in 2024-09-13T14:24:57.350 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:57.029515987 +0000 UTC m=+0.104490585 container create ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:24:57.350 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:56.941279258 +0000 UTC m=+0.016253856 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:57.350 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:57.262425674 +0000 UTC m=+0.337400272 container init ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:24:57.350 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:57.267634074 +0000 UTC m=+0.342608675 container start ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:24:57.350 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:57.348997302 +0000 UTC m=+0.423971918 container attach ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T14:24:57.365 INFO:tasks.rados.rados.0.smithi026.stdout:1768: writing smithi02683976-42 from 8306688 to 8421376 tid 1 2024-09-13T14:24:57.365 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 164 2024-09-13T14:24:57.365 INFO:tasks.rados.rados.0.smithi026.stdout:1762: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:24:57.365 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 802 (ObjNum 574 snap 163 seq_num 574) dirty exists 2024-09-13T14:24:57.365 INFO:tasks.rados.rados.0.smithi026.stdout:1761: finishing write tid 3 to smithi02683976-17 2024-09-13T14:24:57.366 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 799 (ObjNum 577 snap 164 seq_num 577) dirty exists 2024-09-13T14:24:57.366 INFO:tasks.rados.rados.0.smithi026.stdout:1761: left oid 17 (ObjNum 577 snap 164 seq_num 577) 2024-09-13T14:24:57.366 INFO:tasks.rados.rados.0.smithi026.stdout:1769: seq_num 581 ranges {8306688=139264} 2024-09-13T14:24:57.411 INFO:tasks.rados.rados.0.smithi026.stdout:1769: writing smithi02683976-24 from 8306688 to 8445952 tid 1 2024-09-13T14:24:57.411 INFO:tasks.rados.rados.0.smithi026.stdout:1761: done (7 left) 2024-09-13T14:24:57.411 INFO:tasks.rados.rados.0.smithi026.stdout:1762: done (6 left) 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 164 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1764: finishing write tid 1 to smithi02683976-41 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1764: finishing write tid 2 to smithi02683976-41 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1764: finishing write tid 3 to smithi02683976-41 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 804 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1764: left oid 41 (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1767: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:24:57.412 INFO:tasks.rados.rados.0.smithi026.stdout:1770: seq_num 582 ranges {3997696=155648} 2024-09-13T14:24:57.437 INFO:tasks.rados.rados.0.smithi026.stdout:1770: writing smithi02683976-7 from 3997696 to 4153344 tid 1 2024-09-13T14:24:57.437 INFO:tasks.rados.rados.0.smithi026.stdout:1764: done (6 left) 2024-09-13T14:24:57.437 INFO:tasks.rados.rados.0.smithi026.stdout:1771: snap_remove snap 162 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:24:57.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:24:57.667 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:24:58.153 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:24:58.153 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:24:58.153 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:24:58.153 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:24:58.153 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate[115140]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 bash[115125]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:24:58.154 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:57 smithi026 podman[115125]: 2024-09-13 14:24:57.832742405 +0000 UTC m=+0.907717007 container died ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1765: finishing write tid 1 to smithi02683976-12 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1765: finishing write tid 2 to smithi02683976-12 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1767: finishing copy_from to smithi02683976-29 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 798 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1765: finishing write tid 3 to smithi02683976-12 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 932 (ObjNum 579 snap 164 seq_num 579) dirty exists 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1765: left oid 12 (ObjNum 579 snap 164 seq_num 579) 2024-09-13T14:24:58.213 INFO:tasks.rados.rados.0.smithi026.stdout:1766: expect (ObjNum 534 snap 149 seq_num 534) 2024-09-13T14:24:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[99461]: pgmap v201: 121 pgs: 1 active+clean+snaptrim, 24 active+undersized, 2 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 181 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 17 op/s; 20/249 objects degraded (8.032%) 2024-09-13T14:24:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 20/249 objects degraded (8.032%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[99461]: osdmap e403: 8 total, 7 up, 8 in 2024-09-13T14:24:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[97022]: pgmap v201: 121 pgs: 1 active+clean+snaptrim, 24 active+undersized, 2 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 181 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 17 op/s; 20/249 objects degraded (8.032%) 2024-09-13T14:24:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 20/249 objects degraded (8.032%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-mon[97022]: osdmap e403: 8 total, 7 up, 8 in 2024-09-13T14:24:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:58 smithi145 ceph-mon[89145]: pgmap v201: 121 pgs: 1 active+clean+snaptrim, 24 active+undersized, 2 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 181 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 17 op/s; 20/249 objects degraded (8.032%) 2024-09-13T14:24:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:58 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 20/249 objects degraded (8.032%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:24:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:58 smithi145 ceph-mon[89145]: osdmap e403: 8 total, 7 up, 8 in 2024-09-13T14:24:58.459 INFO:tasks.rados.rados.0.smithi026.stdout:1768: finishing write tid 1 to smithi02683976-42 2024-09-13T14:24:58.459 INFO:tasks.rados.rados.0.smithi026.stdout:1768: finishing write tid 2 to smithi02683976-42 2024-09-13T14:24:58.459 INFO:tasks.rados.rados.0.smithi026.stdout:1768: finishing write tid 3 to smithi02683976-42 2024-09-13T14:24:58.459 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 452 (ObjNum 580 snap 164 seq_num 580) dirty exists 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1768: left oid 42 (ObjNum 580 snap 164 seq_num 580) 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1769: finishing write tid 1 to smithi02683976-24 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1769: finishing write tid 2 to smithi02683976-24 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1769: finishing write tid 3 to smithi02683976-24 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 631 (ObjNum 581 snap 164 seq_num 581) dirty exists 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1769: left oid 24 (ObjNum 581 snap 164 seq_num 581) 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1770: finishing write tid 1 to smithi02683976-7 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1770: finishing write tid 2 to smithi02683976-7 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1770: finishing write tid 3 to smithi02683976-7 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 525 (ObjNum 582 snap 164 seq_num 582) dirty exists 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1770: left oid 7 (ObjNum 582 snap 164 seq_num 582) 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1765: done (6 left) 2024-09-13T14:24:58.460 INFO:tasks.rados.rados.0.smithi026.stdout:1766: done (5 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1767: done (4 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1768: done (3 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1769: done (2 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1770: done (1 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1771: done (0 left) 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1772: copy_from oid 39 from oid 4 current snap is 164 2024-09-13T14:24:58.461 INFO:tasks.rados.rados.0.smithi026.stdout:1773: snap_create 2024-09-13T14:24:58.473 INFO:tasks.rados.rados.0.smithi026.stdout:1772: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:24:58.474 INFO:tasks.rados.rados.0.smithi026.stdout:1772: finishing copy_from to smithi02683976-39 2024-09-13T14:24:58.474 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 587 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:24:58.778 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 podman[115281]: 2024-09-13 14:24:58.436599004 +0000 UTC m=+0.597171139 container remove ecca82492a07ae3527ee98cca38e331b4a29ea1062c3a77fb60d679e40d4542e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T14:24:58.779 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 podman[115347]: 2024-09-13 14:24:58.54796735 +0000 UTC m=+0.016678023 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:58.779 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 podman[115347]: 2024-09-13 14:24:58.671250615 +0000 UTC m=+0.139961276 container create 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:24:59.216 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 podman[115347]: 2024-09-13 14:24:58.94867029 +0000 UTC m=+0.417380951 container init 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T14:24:59.217 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 podman[115347]: 2024-09-13 14:24:58.954365507 +0000 UTC m=+0.423076180 container start 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T14:24:59.217 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:58 smithi026 ceph-osd[115385]: -- 172.21.15.26:0/3737797538 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559fdf317a40 con 0x559fdf2f5c00 2024-09-13T14:24:59.217 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:59 smithi026 bash[115347]: 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 2024-09-13T14:24:59.217 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:24:59 smithi026 systemd[1]: Started Ceph osd.2 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:1772: done (1 left) 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:1773: done (0 left) 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:1774: read oid 44 snap -1 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:1774: expect (ObjNum 559 snap 160 seq_num 559) 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 165 2024-09-13T14:24:59.219 INFO:tasks.rados.rados.0.smithi026.stdout:1775: seq_num 583 ranges {4153344=172032} 2024-09-13T14:24:59.244 INFO:tasks.rados.rados.0.smithi026.stdout:1775: writing smithi02683976-7 from 4153344 to 4325376 tid 1 2024-09-13T14:24:59.244 INFO:tasks.rados.rados.0.smithi026.stdout:1776: delete oid 12 current snap is 165 2024-09-13T14:24:59.251 INFO:tasks.rados.rados.0.smithi026.stdout:1775: finishing write tid 1 to smithi02683976-7 2024-09-13T14:24:59.251 INFO:tasks.rados.rados.0.smithi026.stdout:1775: finishing write tid 2 to smithi02683976-7 2024-09-13T14:24:59.252 INFO:tasks.rados.rados.0.smithi026.stdout:1775: finishing write tid 3 to smithi02683976-7 2024-09-13T14:24:59.253 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 527 (ObjNum 583 snap 165 seq_num 583) dirty exists 2024-09-13T14:24:59.253 INFO:tasks.rados.rados.0.smithi026.stdout:1775: left oid 7 (ObjNum 583 snap 165 seq_num 583) 2024-09-13T14:24:59.270 INFO:tasks.rados.rados.0.smithi026.stdout:1775: done (2 left) 2024-09-13T14:24:59.270 INFO:tasks.rados.rados.0.smithi026.stdout:1776: done (1 left) 2024-09-13T14:24:59.270 INFO:tasks.rados.rados.0.smithi026.stdout:1777: rollback oid 20 current snap is 165 2024-09-13T14:24:59.270 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 163 2024-09-13T14:24:59.270 INFO:tasks.rados.rados.0.smithi026.stdout:1778: snap_remove snap 164 2024-09-13T14:24:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[97022]: osdmap e404: 8 total, 7 up, 8 in 2024-09-13T14:24:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[99461]: osdmap e404: 8 total, 7 up, 8 in 2024-09-13T14:24:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:24:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:59 smithi145 ceph-mon[89145]: osdmap e404: 8 total, 7 up, 8 in 2024-09-13T14:24:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:24:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:24:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:00.224 INFO:tasks.rados.rados.0.smithi026.stdout:1778: done (2 left) 2024-09-13T14:25:00.224 INFO:tasks.rados.rados.0.smithi026.stdout:1779: delete oid 27 current snap is 165 2024-09-13T14:25:00.225 INFO:tasks.rados.rados.0.smithi026.stdout:1777: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:25:00.225 INFO:tasks.rados.rados.0.smithi026.stdout:1777: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:25:00.225 INFO:tasks.rados.rados.0.smithi026.stdout:1777: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:25:00.225 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 949 (ObjNum 570 snap 163 seq_num 570) dirty exists 2024-09-13T14:25:00.225 INFO:tasks.rados.rados.0.smithi026.stdout:1774: expect (ObjNum 559 snap 160 seq_num 559) 2024-09-13T14:25:00.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:00 smithi026 ceph-mon[99461]: pgmap v204: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.0 MiB/s wr, 25 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:00.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:00 smithi026 ceph-mon[99461]: osdmap e405: 8 total, 7 up, 8 in 2024-09-13T14:25:00.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:00 smithi026 ceph-mon[97022]: pgmap v204: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.0 MiB/s wr, 25 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:00.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:00 smithi026 ceph-mon[97022]: osdmap e405: 8 total, 7 up, 8 in 2024-09-13T14:25:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:00 smithi145 ceph-mon[89145]: pgmap v204: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.0 MiB/s wr, 25 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:00.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:00 smithi145 ceph-mon[89145]: osdmap e405: 8 total, 7 up, 8 in 2024-09-13T14:25:00.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:00.897 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:25:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:25:00.597+0000 7f452ade9640 -1 Falling back to public interface 2024-09-13T14:25:01.108 INFO:tasks.rados.rados.0.smithi026.stdout:1774: done (2 left) 2024-09-13T14:25:01.108 INFO:tasks.rados.rados.0.smithi026.stdout:1777: done (1 left) 2024-09-13T14:25:01.108 INFO:tasks.rados.rados.0.smithi026.stdout:1779: done (0 left) 2024-09-13T14:25:01.108 INFO:tasks.rados.rados.0.smithi026.stdout:1780: snap_create 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1780: done (0 left) 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1781: read oid 27 snap -1 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1781: expect deleted 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1782: copy_from oid 49 from oid 13 current snap is 166 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1783: copy_from oid 17 from oid 41 current snap is 166 2024-09-13T14:25:01.230 INFO:tasks.rados.rados.0.smithi026.stdout:1784: read oid 25 snap 163 2024-09-13T14:25:01.231 INFO:tasks.rados.rados.0.smithi026.stdout:1784: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:01.231 INFO:tasks.rados.rados.0.smithi026.stdout:1785: snap_create 2024-09-13T14:25:01.236 INFO:tasks.rados.rados.0.smithi026.stdout:1782: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:25:01.240 INFO:tasks.rados.rados.0.smithi026.stdout:1782: finishing copy_from to smithi02683976-49 2024-09-13T14:25:01.240 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 798 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-13T14:25:01.248 INFO:tasks.rados.rados.0.smithi026.stdout:1784: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:01.448 INFO:tasks.rados.rados.0.smithi026.stdout:1783: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:25:01.448 INFO:tasks.rados.rados.0.smithi026.stdout:1783: finishing copy_from to smithi02683976-17 2024-09-13T14:25:01.448 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 804 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:01.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:01 smithi026 ceph-mon[99461]: osdmap e406: 8 total, 7 up, 8 in 2024-09-13T14:25:01.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:01 smithi026 ceph-mon[97022]: osdmap e406: 8 total, 7 up, 8 in 2024-09-13T14:25:01.560 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:01] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-13T14:25:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:01 smithi145 ceph-mon[89145]: osdmap e406: 8 total, 7 up, 8 in 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1781: done (4 left) 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1782: done (3 left) 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1783: done (2 left) 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1784: done (1 left) 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1785: done (0 left) 2024-09-13T14:25:02.235 INFO:tasks.rados.rados.0.smithi026.stdout:1786: rmattr oid 13 current snap is 167 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1787: read oid 26 snap 165 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1787: expect (ObjNum 517508784 snap 21981 seq_num 519045728) 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1788: read oid 4 snap 166 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1788: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1789: rmattr oid 17 current snap is 167 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1789: done (3 left) 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1790: rollback oid 47 current snap is 167 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 163 2024-09-13T14:25:02.236 INFO:tasks.rados.rados.0.smithi026.stdout:1791: setattr oid 28 current snap is 167 2024-09-13T14:25:02.237 INFO:tasks.rados.rados.0.smithi026.stdout:1792: read oid 19 snap -1 2024-09-13T14:25:02.237 INFO:tasks.rados.rados.0.smithi026.stdout:1792: expect deleted 2024-09-13T14:25:02.237 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 167 2024-09-13T14:25:02.237 INFO:tasks.rados.rados.0.smithi026.stdout:1793: seq_num 586 ranges {49152=221184} 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:1793: writing smithi02683976-32 from 49152 to 270336 tid 1 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:1794: copy_from oid 5 from oid 23 current snap is 167 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:1795: rollback oid 33 current snap is 167 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 163 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:1796: copy_from oid 3 from oid 11 current snap is 167 2024-09-13T14:25:02.239 INFO:tasks.rados.rados.0.smithi026.stdout:1797: read oid 22 snap 166 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1797: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1798: read oid 25 snap -1 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1798: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1799: read oid 7 snap -1 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1799: expect (ObjNum 583 snap 165 seq_num 583) 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1800: rollback oid 30 current snap is 167 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 163 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1801: copy_from oid 37 from oid 21 current snap is 167 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1802: read oid 45 snap -1 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1802: expect (ObjNum 571 snap 163 seq_num 571) 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:1792: done (15 left) 2024-09-13T14:25:02.240 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 167 2024-09-13T14:25:02.241 INFO:tasks.rados.rados.0.smithi026.stdout:1803: seq_num 587 ranges {0=237568} 2024-09-13T14:25:02.241 INFO:tasks.rados.rados.0.smithi026.stdout:1803: writing smithi02683976-40 from 0 to 237568 tid 1 2024-09-13T14:25:02.241 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 16 2024-09-13T14:25:02.242 INFO:tasks.rados.rados.0.smithi026.stdout:1787: expect (ObjNum 517508784 snap 21981 seq_num 519045728) 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 950 (ObjNum 129 snap 0 seq_num 2279407508) dirty exists 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1800: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 0 (ObjNum 33 snap 0 seq_num 517479680) dirty dne 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1786: done (15 left) 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1787: done (14 left) 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1800: done (13 left) 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1804: rollback oid 20 current snap is 167 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 163 2024-09-13T14:25:02.245 INFO:tasks.rados.rados.0.smithi026.stdout:1805: snap_create 2024-09-13T14:25:02.246 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 809 (ObjNum 530 snap 148 seq_num 530) dirty exists 2024-09-13T14:25:02.251 INFO:tasks.rados.rados.0.smithi026.stdout:1803: finishing write tid 1 to smithi02683976-40 2024-09-13T14:25:02.251 INFO:tasks.rados.rados.0.smithi026.stdout:1803: finishing write tid 2 to smithi02683976-40 2024-09-13T14:25:02.252 INFO:tasks.rados.rados.0.smithi026.stdout:1795: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:25:02.254 INFO:tasks.rados.rados.0.smithi026.stdout:1804: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:25:02.255 INFO:tasks.rados.rados.0.smithi026.stdout:1795: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:25:02.257 INFO:tasks.rados.rados.0.smithi026.stdout:1793: finishing write tid 1 to smithi02683976-32 2024-09-13T14:25:02.257 INFO:tasks.rados.rados.0.smithi026.stdout:1793: finishing write tid 2 to smithi02683976-32 2024-09-13T14:25:02.257 INFO:tasks.rados.rados.0.smithi026.stdout:1804: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:25:02.260 INFO:tasks.rados.rados.0.smithi026.stdout:1803: finishing write tid 3 to smithi02683976-40 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 811 (ObjNum 587 snap 167 seq_num 587) dirty exists 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:1803: left oid 40 (ObjNum 587 snap 167 seq_num 587) 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:1790: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:1790: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:1790: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:25:02.261 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 723 (ObjNum 548 snap 157 seq_num 548) dirty exists 2024-09-13T14:25:02.262 INFO:tasks.rados.rados.0.smithi026.stdout:1799: expect (ObjNum 583 snap 165 seq_num 583) 2024-09-13T14:25:02.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:02 smithi026 ceph-mon[99461]: pgmap v207: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 8 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:02.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:02 smithi026 ceph-mon[99461]: osdmap e407: 8 total, 7 up, 8 in 2024-09-13T14:25:02.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:02 smithi026 ceph-mon[97022]: pgmap v207: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 8 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:02.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:02 smithi026 ceph-mon[97022]: osdmap e407: 8 total, 7 up, 8 in 2024-09-13T14:25:02.556 INFO:tasks.rados.rados.0.smithi026.stdout:1795: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:25:02.556 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 939 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:25:02.556 INFO:tasks.rados.rados.0.smithi026.stdout:1804: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:25:02.556 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 953 (ObjNum 570 snap 163 seq_num 570) dirty exists 2024-09-13T14:25:02.556 INFO:tasks.rados.rados.0.smithi026.stdout:1798: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:02 smithi145 ceph-mon[89145]: pgmap v207: 121 pgs: 1 active+clean+snaptrim, 27 active+undersized, 10 active+undersized+degraded, 83 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 8 op/s; 31/240 objects degraded (12.917%) 2024-09-13T14:25:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:02 smithi145 ceph-mon[89145]: osdmap e407: 8 total, 7 up, 8 in 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:1801: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:1801: finishing copy_from to smithi02683976-37 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 920 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:1796: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:1796: finishing copy_from to smithi02683976-3 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 905 (ObjNum 563 snap 161 seq_num 563) dirty exists 2024-09-13T14:25:02.751 INFO:tasks.rados.rados.0.smithi026.stdout:1788: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:25:02.900 INFO:tasks.rados.rados.0.smithi026.stdout:1794: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:25:02.900 INFO:tasks.rados.rados.0.smithi026.stdout:1794: finishing copy_from to smithi02683976-5 2024-09-13T14:25:02.900 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 575 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:25:02.900 INFO:tasks.rados.rados.0.smithi026.stdout:1797: expect (ObjNum 216 snap 53 seq_num 216) 2024-09-13T14:25:03.241 INFO:tasks.rados.rados.0.smithi026.stdout:1793: finishing write tid 3 to smithi02683976-32 2024-09-13T14:25:03.241 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 800 (ObjNum 586 snap 167 seq_num 586) dirty exists 2024-09-13T14:25:03.241 INFO:tasks.rados.rados.0.smithi026.stdout:1793: left oid 32 (ObjNum 586 snap 167 seq_num 586) 2024-09-13T14:25:03.241 INFO:tasks.rados.rados.0.smithi026.stdout:1802: expect (ObjNum 571 snap 163 seq_num 571) 2024-09-13T14:25:03.528 INFO:tasks.rados.rados.0.smithi026.stdout:1788: done (14 left) 2024-09-13T14:25:03.528 INFO:tasks.rados.rados.0.smithi026.stdout:1790: done (13 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1791: done (12 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1793: done (11 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1794: done (10 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1795: done (9 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1796: done (8 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1797: done (7 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1798: done (6 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1799: done (5 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1801: done (4 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1802: done (3 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1803: done (2 left) 2024-09-13T14:25:03.529 INFO:tasks.rados.rados.0.smithi026.stdout:1804: done (1 left) 2024-09-13T14:25:03.530 INFO:tasks.rados.rados.0.smithi026.stdout:1805: done (0 left) 2024-09-13T14:25:03.530 INFO:tasks.rados.rados.0.smithi026.stdout:1806: rmattr oid 18 current snap is 168 2024-09-13T14:25:03.530 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 168 2024-09-13T14:25:03.530 INFO:tasks.rados.rados.0.smithi026.stdout:1807: seq_num 588 ranges {6750208=253952} 2024-09-13T14:25:03.566 INFO:tasks.rados.rados.0.smithi026.stdout:1807: writing smithi02683976-14 from 6750208 to 7004160 tid 1 2024-09-13T14:25:03.567 INFO:tasks.rados.rados.0.smithi026.stdout:1808: snap_create 2024-09-13T14:25:03.567 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 802 (ObjNum 547 snap 156 seq_num 547) dirty exists 2024-09-13T14:25:03.569 INFO:tasks.rados.rados.0.smithi026.stdout:1807: finishing write tid 1 to smithi02683976-14 2024-09-13T14:25:03.569 INFO:tasks.rados.rados.0.smithi026.stdout:1807: finishing write tid 2 to smithi02683976-14 2024-09-13T14:25:03.570 INFO:tasks.rados.rados.0.smithi026.stdout:1807: finishing write tid 3 to smithi02683976-14 2024-09-13T14:25:03.570 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 805 (ObjNum 588 snap 168 seq_num 588) dirty exists 2024-09-13T14:25:03.571 INFO:tasks.rados.rados.0.smithi026.stdout:1807: left oid 14 (ObjNum 588 snap 168 seq_num 588) 2024-09-13T14:25:03.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[99461]: osdmap e408: 8 total, 7 up, 8 in 2024-09-13T14:25:03.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[97022]: osdmap e408: 8 total, 7 up, 8 in 2024-09-13T14:25:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:03 smithi145 ceph-mon[89145]: osdmap e408: 8 total, 7 up, 8 in 2024-09-13T14:25:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1806: done (2 left) 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1807: done (1 left) 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1808: done (0 left) 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1809: copy_from oid 34 from oid 41 current snap is 169 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1810: rollback oid 2 current snap is 169 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 166 2024-09-13T14:25:03.917 INFO:tasks.rados.rados.0.smithi026.stdout:1811: rollback oid 26 current snap is 169 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 167 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1812: read oid 9 snap -1 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1812: expect (ObjNum 572 snap 163 seq_num 572) 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1813: read oid 16 snap -1 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1813: expect (ObjNum 518 snap 142 seq_num 518) 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 169 2024-09-13T14:25:03.918 INFO:tasks.rados.rados.0.smithi026.stdout:1814: seq_num 589 ranges {8421376=270336} 2024-09-13T14:25:03.966 INFO:tasks.rados.rados.0.smithi026.stdout:1814: writing smithi02683976-42 from 8421376 to 8691712 tid 1 2024-09-13T14:25:03.966 INFO:tasks.rados.rados.0.smithi026.stdout:1815: snap_remove snap 165 2024-09-13T14:25:04.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 34/264 objects degraded (12.879%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: osdmap e409: 8 total, 7 up, 8 in 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[97022]: osdmap e410: 8 total, 7 up, 8 in 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 34/264 objects degraded (12.879%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: osdmap e409: 8 total, 7 up, 8 in 2024-09-13T14:25:04.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-mon[99461]: osdmap e410: 8 total, 7 up, 8 in 2024-09-13T14:25:04.363 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:25:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:25:04.260+0000 7f452ade9640 -1 osd.2 399 log_to_monitors true 2024-09-13T14:25:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 34/264 objects degraded (12.879%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: osdmap e409: 8 total, 7 up, 8 in 2024-09-13T14:25:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:04.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-mon[89145]: osdmap e410: 8 total, 7 up, 8 in 2024-09-13T14:25:04.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:04.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:25:04.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:04.687 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:25:04.920 INFO:tasks.rados.rados.0.smithi026.stdout:1811: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:25:04.920 INFO:tasks.rados.rados.0.smithi026.stdout:1810: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:1810: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:1810: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 634 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:1811: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:1811: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 936 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-13T14:25:04.921 INFO:tasks.rados.rados.0.smithi026.stdout:1812: expect (ObjNum 572 snap 163 seq_num 572) 2024-09-13T14:25:04.991 INFO:tasks.rados.rados.0.smithi026.stdout:1813: expect (ObjNum 518 snap 142 seq_num 518) 2024-09-13T14:25:05.256 INFO:tasks.rados.rados.0.smithi026.stdout:1809: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:25:05.256 INFO:tasks.rados.rados.0.smithi026.stdout:1809: finishing copy_from to smithi02683976-34 2024-09-13T14:25:05.256 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 804 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:05.256 INFO:tasks.rados.rados.0.smithi026.stdout:1814: finishing write tid 1 to smithi02683976-42 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1814: finishing write tid 2 to smithi02683976-42 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1814: finishing write tid 3 to smithi02683976-42 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 922 (ObjNum 589 snap 169 seq_num 589) dirty exists 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1814: left oid 42 (ObjNum 589 snap 169 seq_num 589) 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1809: done (6 left) 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1810: done (5 left) 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1811: done (4 left) 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1812: done (3 left) 2024-09-13T14:25:05.257 INFO:tasks.rados.rados.0.smithi026.stdout:1813: done (2 left) 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1814: done (1 left) 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1815: done (0 left) 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1816: copy_from oid 23 from oid 1 current snap is 169 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1817: rmattr oid 25 current snap is 169 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1818: setattr oid 43 current snap is 169 2024-09-13T14:25:05.258 INFO:tasks.rados.rados.0.smithi026.stdout:1819: snap_remove snap 163 2024-09-13T14:25:05.345 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[99461]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[99461]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[99461]: osdmap e411: 8 total, 7 up, 8 in 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[99461]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[99461]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[97022]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[97022]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[97022]: osdmap e411: 8 total, 7 up, 8 in 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[97022]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:05.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-mon[97022]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' 2024-09-13T14:25:05.347 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:25:05 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:25:05.209+0000 7f451eb8b640 -1 osd.2 399 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 2s ago 8m 19.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (3m) 111s ago 8m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (9m) 2s ago 9m 15.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (9m) 111s ago 9m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (2m) 111s ago 13m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:25:05.384 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (3m) 2s ago 15m 522M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (2m) 2s ago 15m 53.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (116s) 111s ago 13m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 2s ago 13m 45.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 2s ago 8m 11.9M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (3m) 111s ago 8m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (74s) 2s ago 13m 128M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (40s) 2s ago 12m 104M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (6s) 2s ago 12m 11.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (11m) 2s ago 11m 114M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 8e148473b9fc 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (11m) 111s ago 11m 171M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (11m) 111s ago 11m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (10m) 111s ago 10m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:25:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (10m) 111s ago 10m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:25:05.386 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (2m) 111s ago 8m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:25:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-mon[89145]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:25:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-mon[89145]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:25:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-mon[89145]: osdmap e411: 8 total, 7 up, 8 in 2024-09-13T14:25:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-mon[89145]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-mon[89145]: from='osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581]' entity='osd.2' 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:05.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 7, 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:25:05.736 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:25:06.035 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "8/20 daemons upgraded", 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:25:06.036 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:25:06.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:06.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:1819: done (3 left) 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:1820: setattr oid 47 current snap is 169 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:1816: finishing copy_from to smithi02683976-23 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:1816: got expected ENOENT (src dne) 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:1816: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 635 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:25:06.206 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 954 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:25:06.209 INFO:tasks.rados.rados.0.smithi026.stdout:1816: done (3 left) 2024-09-13T14:25:06.209 INFO:tasks.rados.rados.0.smithi026.stdout:1817: done (2 left) 2024-09-13T14:25:06.209 INFO:tasks.rados.rados.0.smithi026.stdout:1818: done (1 left) 2024-09-13T14:25:06.209 INFO:tasks.rados.rados.0.smithi026.stdout:1821: copy_from oid 37 from oid 1 current snap is 169 2024-09-13T14:25:06.209 INFO:tasks.rados.rados.0.smithi026.stdout:1822: rollback oid 15 current snap is 169 2024-09-13T14:25:06.210 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 166 2024-09-13T14:25:06.210 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 169 2024-09-13T14:25:06.210 INFO:tasks.rados.rados.0.smithi026.stdout:1823: seq_num 590 ranges {4882432=286720} 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1823: writing smithi02683976-22 from 4882432 to 5169152 tid 1 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1824: rollback oid 10 current snap is 169 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 167 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1822: finishing rollback tid 0 to smithi02683976-15 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1822: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1822: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 590 (ObjNum 551 snap 158 seq_num 551) dirty exists 2024-09-13T14:25:06.237 INFO:tasks.rados.rados.0.smithi026.stdout:1822: done (4 left) 2024-09-13T14:25:06.238 INFO:tasks.rados.rados.0.smithi026.stdout:1825: snap_create 2024-09-13T14:25:06.239 INFO:tasks.rados.rados.0.smithi026.stdout:1823: finishing write tid 1 to smithi02683976-22 2024-09-13T14:25:06.239 INFO:tasks.rados.rados.0.smithi026.stdout:1823: finishing write tid 2 to smithi02683976-22 2024-09-13T14:25:06.241 INFO:tasks.rados.rados.0.smithi026.stdout:1823: finishing write tid 3 to smithi02683976-22 2024-09-13T14:25:06.241 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 938 (ObjNum 590 snap 169 seq_num 590) dirty exists 2024-09-13T14:25:06.241 INFO:tasks.rados.rados.0.smithi026.stdout:1823: left oid 22 (ObjNum 590 snap 169 seq_num 590) 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3765013344' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581] boot 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:25:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3765013344' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581] boot 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:25:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:25:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 18 op/s; 34/264 objects degraded (12.879%) 2024-09-13T14:25:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3765013344' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: osd.2 [v2:172.21.15.26:6818/718889581,v1:172.21.15.26:6819/718889581] boot 2024-09-13T14:25:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:25:06.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:25:07.168 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:06 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:25:06.962+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:25:07.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:1823: done (4 left) 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:1825: done (3 left) 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 170 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:1826: seq_num 591 ranges {0=303104} 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:1826: writing smithi02683976-8 from 0 to 303104 tid 1 2024-09-13T14:25:07.213 INFO:tasks.rados.rados.0.smithi026.stdout:1827: delete oid 49 current snap is 170 2024-09-13T14:25:07.218 INFO:tasks.rados.rados.0.smithi026.stdout:1826: finishing write tid 1 to smithi02683976-8 2024-09-13T14:25:07.218 INFO:tasks.rados.rados.0.smithi026.stdout:1826: finishing write tid 2 to smithi02683976-8 2024-09-13T14:25:07.219 INFO:tasks.rados.rados.0.smithi026.stdout:1826: finishing write tid 3 to smithi02683976-8 2024-09-13T14:25:07.219 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 586 (ObjNum 591 snap 170 seq_num 591) dirty exists 2024-09-13T14:25:07.220 INFO:tasks.rados.rados.0.smithi026.stdout:1826: left oid 8 (ObjNum 591 snap 170 seq_num 591) 2024-09-13T14:25:07.224 INFO:tasks.rados.rados.0.smithi026.stdout:1824: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:25:07.227 INFO:tasks.rados.rados.0.smithi026.stdout:1824: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:25:07.227 INFO:tasks.rados.rados.0.smithi026.stdout:1824: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:25:07.228 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 814 (ObjNum 129 snap 0 seq_num 3890191696) dirty exists 2024-09-13T14:25:07.232 INFO:tasks.rados.rados.0.smithi026.stdout:1824: done (4 left) 2024-09-13T14:25:07.232 INFO:tasks.rados.rados.0.smithi026.stdout:1826: done (3 left) 2024-09-13T14:25:07.232 INFO:tasks.rados.rados.0.smithi026.stdout:1827: done (2 left) 2024-09-13T14:25:07.232 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 170 2024-09-13T14:25:07.232 INFO:tasks.rados.rados.0.smithi026.stdout:1828: seq_num 593 ranges {2154496=335872} 2024-09-13T14:25:07.248 INFO:tasks.rados.rados.0.smithi026.stdout:1828: writing smithi02683976-50 from 2154496 to 2490368 tid 1 2024-09-13T14:25:07.248 INFO:tasks.rados.rados.0.smithi026.stdout:1829: delete oid 29 current snap is 170 2024-09-13T14:25:07.248 INFO:tasks.rados.rados.0.smithi026.stdout:1821: finishing copy_from to smithi02683976-37 2024-09-13T14:25:07.248 INFO:tasks.rados.rados.0.smithi026.stdout:1821: got expected ENOENT (src dne) 2024-09-13T14:25:07.248 INFO:tasks.rados.rados.0.smithi026.stdout:1821: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:25:07.251 INFO:tasks.rados.rados.0.smithi026.stdout:1821: done (3 left) 2024-09-13T14:25:07.251 INFO:tasks.rados.rados.0.smithi026.stdout:1829: done (2 left) 2024-09-13T14:25:07.251 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 170 2024-09-13T14:25:07.251 INFO:tasks.rados.rados.0.smithi026.stdout:1828: finishing write tid 1 to smithi02683976-50 2024-09-13T14:25:07.252 INFO:tasks.rados.rados.0.smithi026.stdout:1830: seq_num 595 ranges {2392064=368640} 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:1830: writing smithi02683976-9 from 2392064 to 2760704 tid 1 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:1831: rmattr oid 25 current snap is 170 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:1828: finishing write tid 2 to smithi02683976-50 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:1828: finishing write tid 3 to smithi02683976-50 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 927 (ObjNum 593 snap 170 seq_num 593) dirty exists 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:1828: left oid 50 (ObjNum 593 snap 170 seq_num 593) 2024-09-13T14:25:07.268 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 724 (ObjNum 548 snap 157 seq_num 548) dirty exists 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:1820: done (3 left) 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:1828: done (2 left) 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:1832: copy_from oid 31 from oid 2 current snap is 170 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:1833: copy_from oid 6 from oid 46 current snap is 170 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 170 2024-09-13T14:25:07.269 INFO:tasks.rados.rados.0.smithi026.stdout:1834: seq_num 596 ranges {4513792=385024} 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1834: writing smithi02683976-43 from 4513792 to 4898816 tid 1 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1835: delete oid 47 current snap is 170 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 636 (ObjNum 378 snap 106 seq_num 378) dirty exists 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1830: finishing write tid 1 to smithi02683976-9 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1830: finishing write tid 2 to smithi02683976-9 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1830: finishing write tid 3 to smithi02683976-9 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 930 (ObjNum 595 snap 170 seq_num 595) dirty exists 2024-09-13T14:25:07.297 INFO:tasks.rados.rados.0.smithi026.stdout:1830: left oid 9 (ObjNum 595 snap 170 seq_num 595) 2024-09-13T14:25:07.300 INFO:tasks.rados.rados.0.smithi026.stdout:1832: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:25:07.301 INFO:tasks.rados.rados.0.smithi026.stdout:1833: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:25:07.301 INFO:tasks.rados.rados.0.smithi026.stdout:1833: finishing copy_from to smithi02683976-6 2024-09-13T14:25:07.301 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 455 (ObjNum 542 snap 156 seq_num 542) dirty exists 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1830: done (5 left) 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1831: done (4 left) 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1833: done (3 left) 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1835: done (2 left) 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1836: read oid 16 snap -1 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1836: expect (ObjNum 518 snap 142 seq_num 518) 2024-09-13T14:25:07.302 INFO:tasks.rados.rados.0.smithi026.stdout:1837: copy_from oid 24 from oid 6 current snap is 170 2024-09-13T14:25:07.303 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 170 2024-09-13T14:25:07.303 INFO:tasks.rados.rados.0.smithi026.stdout:1838: seq_num 598 ranges {2244608=417792} 2024-09-13T14:25:07.318 INFO:tasks.rados.rados.0.smithi026.stdout:1838: writing smithi02683976-11 from 2244608 to 2662400 tid 1 2024-09-13T14:25:07.319 INFO:tasks.rados.rados.0.smithi026.stdout:1839: snap_create 2024-09-13T14:25:07.319 INFO:tasks.rados.rados.0.smithi026.stdout:1832: finishing copy_from to smithi02683976-31 2024-09-13T14:25:07.319 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 634 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:25:07.319 INFO:tasks.rados.rados.0.smithi026.stdout:1836: expect (ObjNum 518 snap 142 seq_num 518) 2024-09-13T14:25:07.534 INFO:tasks.rados.rados.0.smithi026.stdout:1834: finishing write tid 1 to smithi02683976-43 2024-09-13T14:25:07.534 INFO:tasks.rados.rados.0.smithi026.stdout:1834: finishing write tid 2 to smithi02683976-43 2024-09-13T14:25:07.534 INFO:tasks.rados.rados.0.smithi026.stdout:1834: finishing write tid 3 to smithi02683976-43 2024-09-13T14:25:07.534 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 957 (ObjNum 596 snap 170 seq_num 596) dirty exists 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1834: left oid 43 (ObjNum 596 snap 170 seq_num 596) 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1838: finishing write tid 1 to smithi02683976-11 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1838: finishing write tid 2 to smithi02683976-11 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1838: finishing write tid 3 to smithi02683976-11 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 936 (ObjNum 598 snap 170 seq_num 598) dirty exists 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1838: left oid 11 (ObjNum 598 snap 170 seq_num 598) 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1837: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:1837: finishing copy_from to smithi02683976-24 2024-09-13T14:25:07.535 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 455 (ObjNum 542 snap 156 seq_num 542) dirty exists 2024-09-13T14:25:07.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[99461]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:07.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:07 smithi026 ceph-mon[97022]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:25:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:07.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:07 smithi145 ceph-mon[89145]: osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1832: done (5 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1834: done (4 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1836: done (3 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1837: done (2 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1838: done (1 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1839: done (0 left) 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1840: read oid 4 snap -1 2024-09-13T14:25:08.260 INFO:tasks.rados.rados.0.smithi026.stdout:1840: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:25:08.261 INFO:tasks.rados.rados.0.smithi026.stdout:1841: rollback oid 30 current snap is 171 2024-09-13T14:25:08.261 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 168 2024-09-13T14:25:08.261 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 171 2024-09-13T14:25:08.261 INFO:tasks.rados.rados.0.smithi026.stdout:1842: seq_num 599 ranges {0=434176} 2024-09-13T14:25:08.262 INFO:tasks.rados.rados.0.smithi026.stdout:1842: writing smithi02683976-10 from 0 to 434176 tid 1 2024-09-13T14:25:08.262 INFO:tasks.rados.rados.0.smithi026.stdout:1843: copy_from oid 38 from oid 39 current snap is 171 2024-09-13T14:25:08.262 INFO:tasks.rados.rados.0.smithi026.stdout:1844: delete oid 50 current snap is 171 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:1842: finishing write tid 1 to smithi02683976-10 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:1844: done (4 left) 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:1845: rollback oid 15 current snap is 171 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 168 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:1846: snap_create 2024-09-13T14:25:08.271 INFO:tasks.rados.rados.0.smithi026.stdout:1842: finishing write tid 2 to smithi02683976-10 2024-09-13T14:25:08.272 INFO:tasks.rados.rados.0.smithi026.stdout:1842: finishing write tid 3 to smithi02683976-10 2024-09-13T14:25:08.272 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 817 (ObjNum 599 snap 171 seq_num 599) dirty exists 2024-09-13T14:25:08.272 INFO:tasks.rados.rados.0.smithi026.stdout:1842: left oid 10 (ObjNum 599 snap 171 seq_num 599) 2024-09-13T14:25:08.280 INFO:tasks.rados.rados.0.smithi026.stdout:1841: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:25:08.280 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 0 (ObjNum 33 snap 0 seq_num 1126842271) dirty dne 2024-09-13T14:25:08.286 INFO:tasks.rados.rados.0.smithi026.stdout:1840: expect (ObjNum 364 snap 102 seq_num 364) 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:1845: finishing rollback tid 0 to smithi02683976-15 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:1845: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:1845: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 593 (ObjNum 551 snap 158 seq_num 551) dirty exists 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:1843: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:1843: finishing copy_from to smithi02683976-38 2024-09-13T14:25:08.434 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 587 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:25:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[99461]: pgmap v216: 121 pgs: 15 peering, 14 active+undersized, 8 active+undersized+degraded, 84 active+clean; 186 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 32/264 objects degraded (12.121%) 2024-09-13T14:25:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:25:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[97022]: pgmap v216: 121 pgs: 15 peering, 14 active+undersized, 8 active+undersized+degraded, 84 active+clean; 186 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 32/264 objects degraded (12.121%) 2024-09-13T14:25:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:08 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:25:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:08 smithi145 ceph-mon[89145]: pgmap v216: 121 pgs: 15 peering, 14 active+undersized, 8 active+undersized+degraded, 84 active+clean; 186 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 32/264 objects degraded (12.121%) 2024-09-13T14:25:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:08 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:08 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1840: done (5 left) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1841: done (4 left) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1842: done (3 left) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1843: done (2 left) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1845: done (1 left) 2024-09-13T14:25:09.265 INFO:tasks.rados.rados.0.smithi026.stdout:1846: done (0 left) 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1847: read oid 30 snap 169 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1847: expect deleted 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1848: rollback oid 17 current snap is 172 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 166 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1849: read oid 14 snap 168 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1849: expect (ObjNum 588 snap 168 seq_num 588) 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 172 2024-09-13T14:25:09.266 INFO:tasks.rados.rados.0.smithi026.stdout:1850: seq_num 601 ranges {0=475136} 2024-09-13T14:25:09.268 INFO:tasks.rados.rados.0.smithi026.stdout:1850: writing smithi02683976-13 from 0 to 475136 tid 1 2024-09-13T14:25:09.268 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 172 2024-09-13T14:25:09.268 INFO:tasks.rados.rados.0.smithi026.stdout:1851: seq_num 602 ranges {6905856=491520} 2024-09-13T14:25:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:1851: writing smithi02683976-23 from 6905856 to 7397376 tid 1 2024-09-13T14:25:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:1852: rollback oid 32 current snap is 172 2024-09-13T14:25:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 168 2024-09-13T14:25:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:1848: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:25:09.310 INFO:tasks.rados.rados.0.smithi026.stdout:1848: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1847: done (5 left) 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 172 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1848: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 807 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1850: finishing write tid 1 to smithi02683976-13 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1850: finishing write tid 2 to smithi02683976-13 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1850: finishing write tid 3 to smithi02683976-13 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 959 (ObjNum 601 snap 172 seq_num 601) dirty exists 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1850: left oid 13 (ObjNum 601 snap 172 seq_num 601) 2024-09-13T14:25:09.311 INFO:tasks.rados.rados.0.smithi026.stdout:1853: seq_num 603 ranges {4456448=507904} 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1853: writing smithi02683976-6 from 4456448 to 4964352 tid 1 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1848: done (5 left) 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1850: done (4 left) 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1854: read oid 8 snap -1 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1854: expect (ObjNum 591 snap 170 seq_num 591) 2024-09-13T14:25:09.337 INFO:tasks.rados.rados.0.smithi026.stdout:1855: snap_remove snap 170 2024-09-13T14:25:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:09 smithi026 ceph-mon[97022]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:25:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:09 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 32/264 objects degraded (12.121%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:09.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:09 smithi026 ceph-mon[99461]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:25:09.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:09 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 32/264 objects degraded (12.121%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:09 smithi145 ceph-mon[89145]: osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:25:09.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:09 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 32/264 objects degraded (12.121%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:10.270 INFO:tasks.rados.rados.0.smithi026.stdout:1855: done (5 left) 2024-09-13T14:25:10.271 INFO:tasks.rados.rados.0.smithi026.stdout:1856: rollback oid 10 current snap is 172 2024-09-13T14:25:10.271 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 171 2024-09-13T14:25:10.271 INFO:tasks.rados.rados.0.smithi026.stdout:1849: expect (ObjNum 588 snap 168 seq_num 588) 2024-09-13T14:25:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:10 smithi026 ceph-mon[97022]: pgmap v219: 121 pgs: 19 peering, 11 active+undersized, 7 active+undersized+degraded, 84 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 20 op/s; 30/264 objects degraded (11.364%) 2024-09-13T14:25:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:10 smithi026 ceph-mon[97022]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:25:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:10 smithi026 ceph-mon[99461]: pgmap v219: 121 pgs: 19 peering, 11 active+undersized, 7 active+undersized+degraded, 84 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 20 op/s; 30/264 objects degraded (11.364%) 2024-09-13T14:25:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:10 smithi026 ceph-mon[99461]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:25:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:10 smithi145 ceph-mon[89145]: pgmap v219: 121 pgs: 19 peering, 11 active+undersized, 7 active+undersized+degraded, 84 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.1 MiB/s wr, 20 op/s; 30/264 objects degraded (11.364%) 2024-09-13T14:25:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:10 smithi145 ceph-mon[89145]: osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:25:10.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:1851: finishing write tid 1 to smithi02683976-23 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:1851: finishing write tid 2 to smithi02683976-23 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:1852: finishing rollback tid 0 to smithi02683976-32 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:1852: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:1851: finishing write tid 3 to smithi02683976-23 2024-09-13T14:25:10.858 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 818 (ObjNum 602 snap 172 seq_num 602) dirty exists 2024-09-13T14:25:10.859 INFO:tasks.rados.rados.0.smithi026.stdout:1851: left oid 23 (ObjNum 602 snap 172 seq_num 602) 2024-09-13T14:25:10.859 INFO:tasks.rados.rados.0.smithi026.stdout:1852: finishing rollback tid 2 to smithi02683976-32 2024-09-13T14:25:10.859 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 822 (ObjNum 586 snap 167 seq_num 586) dirty exists 2024-09-13T14:25:10.859 INFO:tasks.rados.rados.0.smithi026.stdout:1854: expect (ObjNum 591 snap 170 seq_num 591) 2024-09-13T14:25:10.866 INFO:tasks.rados.rados.0.smithi026.stdout:1853: finishing write tid 1 to smithi02683976-6 2024-09-13T14:25:10.866 INFO:tasks.rados.rados.0.smithi026.stdout:1853: finishing write tid 2 to smithi02683976-6 2024-09-13T14:25:10.866 INFO:tasks.rados.rados.0.smithi026.stdout:1853: finishing write tid 3 to smithi02683976-6 2024-09-13T14:25:10.866 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 941 (ObjNum 603 snap 172 seq_num 603) dirty exists 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1853: left oid 6 (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1856: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1856: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1856: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 820 (ObjNum 599 snap 171 seq_num 599) dirty exists 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1849: done (5 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1851: done (4 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1852: done (3 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1853: done (2 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1854: done (1 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1856: done (0 left) 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1857: read oid 1 snap 167 2024-09-13T14:25:10.867 INFO:tasks.rados.rados.0.smithi026.stdout:1857: expect deleted 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:1858: read oid 3 snap 169 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:1858: expect (ObjNum 563 snap 161 seq_num 563) 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:1859: rmattr oid 39 current snap is 172 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:1859: done (2 left) 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 172 2024-09-13T14:25:10.869 INFO:tasks.rados.rados.0.smithi026.stdout:1860: seq_num 604 ranges {3170304=524288} 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1860: writing smithi02683976-4 from 3170304 to 3694592 tid 1 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1861: read oid 50 snap 167 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1861: expect (ObjNum 558 snap 160 seq_num 558) 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1862: read oid 11 snap -1 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1862: expect (ObjNum 598 snap 170 seq_num 598) 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1863: rmattr oid 20 current snap is 172 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1863: done (5 left) 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1864: rollback oid 44 current snap is 172 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 166 2024-09-13T14:25:10.887 INFO:tasks.rados.rados.0.smithi026.stdout:1858: expect (ObjNum 563 snap 161 seq_num 563) 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1864: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1864: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1860: finishing write tid 1 to smithi02683976-4 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1864: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 463 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1860: finishing write tid 2 to smithi02683976-4 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1860: finishing write tid 3 to smithi02683976-4 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 595 (ObjNum 604 snap 172 seq_num 604) dirty exists 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1860: left oid 4 (ObjNum 604 snap 172 seq_num 604) 2024-09-13T14:25:10.947 INFO:tasks.rados.rados.0.smithi026.stdout:1861: expect (ObjNum 558 snap 160 seq_num 558) 2024-09-13T14:25:11.003 INFO:tasks.rados.rados.0.smithi026.stdout:1862: expect (ObjNum 598 snap 170 seq_num 598) 2024-09-13T14:25:11.109 INFO:tasks.rados.rados.0.smithi026.stdout:1857: done (5 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1858: done (4 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1860: done (3 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1861: done (2 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1862: done (1 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1864: done (0 left) 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 172 2024-09-13T14:25:11.110 INFO:tasks.rados.rados.0.smithi026.stdout:1865: seq_num 605 ranges {0=540672} 2024-09-13T14:25:11.112 INFO:tasks.rados.rados.0.smithi026.stdout:1865: writing smithi02683976-26 from 0 to 540672 tid 1 2024-09-13T14:25:11.112 INFO:tasks.rados.rados.0.smithi026.stdout:1866: snap_create 2024-09-13T14:25:11.116 INFO:tasks.rados.rados.0.smithi026.stdout:1865: finishing write tid 1 to smithi02683976-26 2024-09-13T14:25:11.116 INFO:tasks.rados.rados.0.smithi026.stdout:1865: finishing write tid 2 to smithi02683976-26 2024-09-13T14:25:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:1865: finishing write tid 3 to smithi02683976-26 2024-09-13T14:25:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 940 (ObjNum 605 snap 172 seq_num 605) dirty exists 2024-09-13T14:25:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:1865: left oid 26 (ObjNum 605 snap 172 seq_num 605) 2024-09-13T14:25:11.275 INFO:tasks.rados.rados.0.smithi026.stdout:1865: done (1 left) 2024-09-13T14:25:11.275 INFO:tasks.rados.rados.0.smithi026.stdout:1866: done (0 left) 2024-09-13T14:25:11.275 INFO:tasks.rados.rados.0.smithi026.stdout:1867: delete oid 40 current snap is 173 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1867: done (0 left) 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1868: read oid 17 snap 172 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1868: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1869: read oid 45 snap 171 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1869: expect (ObjNum 571 snap 163 seq_num 571) 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 173 2024-09-13T14:25:11.280 INFO:tasks.rados.rados.0.smithi026.stdout:1870: seq_num 607 ranges {0=573440} 2024-09-13T14:25:11.283 INFO:tasks.rados.rados.0.smithi026.stdout:1870: writing smithi02683976-29 from 0 to 573440 tid 1 2024-09-13T14:25:11.283 INFO:tasks.rados.rados.0.smithi026.stdout:1871: snap_remove snap 168 2024-09-13T14:25:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[97022]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:25:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:11.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:11] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-13T14:25:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[99461]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:25:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:11 smithi145 ceph-mon[89145]: osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:25:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:12.282 INFO:tasks.rados.rados.0.smithi026.stdout:1871: done (3 left) 2024-09-13T14:25:12.282 INFO:tasks.rados.rados.0.smithi026.stdout:1872: delete oid 3 current snap is 173 2024-09-13T14:25:12.282 INFO:tasks.rados.rados.0.smithi026.stdout:1870: finishing write tid 1 to smithi02683976-29 2024-09-13T14:25:12.283 INFO:tasks.rados.rados.0.smithi026.stdout:1870: finishing write tid 2 to smithi02683976-29 2024-09-13T14:25:12.283 INFO:tasks.rados.rados.0.smithi026.stdout:1870: finishing write tid 3 to smithi02683976-29 2024-09-13T14:25:12.283 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 961 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:25:12.283 INFO:tasks.rados.rados.0.smithi026.stdout:1870: left oid 29 (ObjNum 607 snap 173 seq_num 607) 2024-09-13T14:25:12.283 INFO:tasks.rados.rados.0.smithi026.stdout:1869: expect (ObjNum 571 snap 163 seq_num 571) 2024-09-13T14:25:12.573 INFO:tasks.rados.rados.0.smithi026.stdout:1868: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[97022]: pgmap v222: 121 pgs: 19 peering, 4 active+undersized, 98 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.5 MiB/s wr, 26 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/264 objects degraded (11.364%), 7 pgs degraded) 2024-09-13T14:25:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:25:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[97022]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:25:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[99461]: pgmap v222: 121 pgs: 19 peering, 4 active+undersized, 98 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.5 MiB/s wr, 26 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/264 objects degraded (11.364%), 7 pgs degraded) 2024-09-13T14:25:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:25:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:12 smithi026 ceph-mon[99461]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:25:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:12 smithi145 ceph-mon[89145]: pgmap v222: 121 pgs: 19 peering, 4 active+undersized, 98 active+clean; 182 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.5 MiB/s wr, 26 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:12 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/264 objects degraded (11.364%), 7 pgs degraded) 2024-09-13T14:25:12.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:12 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:25:12.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:12 smithi145 ceph-mon[89145]: osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:25:13.388 INFO:tasks.rados.rados.0.smithi026.stdout:1868: done (3 left) 2024-09-13T14:25:13.388 INFO:tasks.rados.rados.0.smithi026.stdout:1869: done (2 left) 2024-09-13T14:25:13.388 INFO:tasks.rados.rados.0.smithi026.stdout:1870: done (1 left) 2024-09-13T14:25:13.388 INFO:tasks.rados.rados.0.smithi026.stdout:1872: done (0 left) 2024-09-13T14:25:13.388 INFO:tasks.rados.rados.0.smithi026.stdout:1873: setattr oid 44 current snap is 173 2024-09-13T14:25:13.390 INFO:tasks.rados.rados.0.smithi026.stdout:1874: rollback oid 18 current snap is 173 2024-09-13T14:25:13.390 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 171 2024-09-13T14:25:13.390 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 173 2024-09-13T14:25:13.390 INFO:tasks.rados.rados.0.smithi026.stdout:1875: seq_num 609 ranges {3170304=606208} 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:1875: writing smithi02683976-48 from 3170304 to 3776512 tid 1 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:1876: read oid 1 snap -1 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:1876: expect deleted 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 173 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 464 (ObjNum 559 snap 160 seq_num 559) dirty exists 2024-09-13T14:25:13.412 INFO:tasks.rados.rados.0.smithi026.stdout:1874: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:25:13.413 INFO:tasks.rados.rados.0.smithi026.stdout:1874: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:25:13.413 INFO:tasks.rados.rados.0.smithi026.stdout:1874: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:25:13.413 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 838 (ObjNum 547 snap 156 seq_num 547) dirty exists 2024-09-13T14:25:13.413 INFO:tasks.rados.rados.0.smithi026.stdout:1877: seq_num 610 ranges {3948544=622592} 2024-09-13T14:25:13.436 INFO:tasks.rados.rados.0.smithi026.stdout:1877: writing smithi02683976-15 from 3948544 to 4571136 tid 1 2024-09-13T14:25:13.436 INFO:tasks.rados.rados.0.smithi026.stdout:1873: done (4 left) 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1874: done (3 left) 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1878: delete oid 35 current snap is 173 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1875: finishing write tid 1 to smithi02683976-48 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1875: finishing write tid 2 to smithi02683976-48 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1875: finishing write tid 3 to smithi02683976-48 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 727 (ObjNum 609 snap 173 seq_num 609) dirty exists 2024-09-13T14:25:13.437 INFO:tasks.rados.rados.0.smithi026.stdout:1875: left oid 48 (ObjNum 609 snap 173 seq_num 609) 2024-09-13T14:25:13.438 INFO:tasks.rados.rados.0.smithi026.stdout:1875: done (3 left) 2024-09-13T14:25:13.438 INFO:tasks.rados.rados.0.smithi026.stdout:1876: done (2 left) 2024-09-13T14:25:13.438 INFO:tasks.rados.rados.0.smithi026.stdout:1878: done (1 left) 2024-09-13T14:25:13.438 INFO:tasks.rados.rados.0.smithi026.stdout:1879: snap_create 2024-09-13T14:25:13.440 INFO:tasks.rados.rados.0.smithi026.stdout:1877: finishing write tid 1 to smithi02683976-15 2024-09-13T14:25:13.440 INFO:tasks.rados.rados.0.smithi026.stdout:1877: finishing write tid 2 to smithi02683976-15 2024-09-13T14:25:13.441 INFO:tasks.rados.rados.0.smithi026.stdout:1877: finishing write tid 3 to smithi02683976-15 2024-09-13T14:25:13.441 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 597 (ObjNum 610 snap 173 seq_num 610) dirty exists 2024-09-13T14:25:13.441 INFO:tasks.rados.rados.0.smithi026.stdout:1877: left oid 15 (ObjNum 610 snap 173 seq_num 610) 2024-09-13T14:25:13.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:13 smithi026 ceph-mon[97022]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:25:13.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:13 smithi026 ceph-mon[99461]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:25:13.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:13 smithi145 ceph-mon[89145]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:25:13.920 INFO:tasks.rados.rados.0.smithi026.stdout:1877: done (1 left) 2024-09-13T14:25:13.920 INFO:tasks.rados.rados.0.smithi026.stdout:1879: done (0 left) 2024-09-13T14:25:13.920 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 174 2024-09-13T14:25:13.920 INFO:tasks.rados.rados.0.smithi026.stdout:1880: seq_num 612 ranges {0=655360} 2024-09-13T14:25:13.923 INFO:tasks.rados.rados.0.smithi026.stdout:1880: writing smithi02683976-3 from 0 to 655360 tid 1 2024-09-13T14:25:13.923 INFO:tasks.rados.rados.0.smithi026.stdout:1881: rollback oid 11 current snap is 174 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 169 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:1882: rollback oid 26 current snap is 174 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 172 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:1883: rollback oid 19 current snap is 174 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 171 2024-09-13T14:25:13.924 INFO:tasks.rados.rados.0.smithi026.stdout:1884: snap_create 2024-09-13T14:25:13.926 INFO:tasks.rados.rados.0.smithi026.stdout:1883: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:25:13.927 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 0 (ObjNum 33 snap 0 seq_num 509658488) dirty dne 2024-09-13T14:25:13.929 INFO:tasks.rados.rados.0.smithi026.stdout:1880: finishing write tid 1 to smithi02683976-3 2024-09-13T14:25:13.930 INFO:tasks.rados.rados.0.smithi026.stdout:1880: finishing write tid 2 to smithi02683976-3 2024-09-13T14:25:13.931 INFO:tasks.rados.rados.0.smithi026.stdout:1880: finishing write tid 3 to smithi02683976-3 2024-09-13T14:25:13.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 960 (ObjNum 612 snap 174 seq_num 612) dirty exists 2024-09-13T14:25:13.931 INFO:tasks.rados.rados.0.smithi026.stdout:1880: left oid 3 (ObjNum 612 snap 174 seq_num 612) 2024-09-13T14:25:13.931 INFO:tasks.rados.rados.0.smithi026.stdout:1881: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:25:13.932 INFO:tasks.rados.rados.0.smithi026.stdout:1881: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:25:13.932 INFO:tasks.rados.rados.0.smithi026.stdout:1881: finishing rollback tid 2 to smithi02683976-11 2024-09-13T14:25:13.932 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 957 (ObjNum 563 snap 161 seq_num 563) dirty exists 2024-09-13T14:25:13.932 INFO:tasks.rados.rados.0.smithi026.stdout:1882: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:25:13.933 INFO:tasks.rados.rados.0.smithi026.stdout:1882: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:25:13.933 INFO:tasks.rados.rados.0.smithi026.stdout:1882: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:25:13.933 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 943 (ObjNum 605 snap 172 seq_num 605) dirty exists 2024-09-13T14:25:14.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:14 smithi145 ceph-mon[89145]: pgmap v225: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 8.6 MiB/s wr, 40 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:14.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:14 smithi145 ceph-mon[89145]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:25:14.457 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:14.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:14 smithi026 ceph-mon[97022]: pgmap v225: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 8.6 MiB/s wr, 40 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:14.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:14 smithi026 ceph-mon[97022]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:25:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:14 smithi026 ceph-mon[99461]: pgmap v225: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 8.6 MiB/s wr, 40 op/s; 27 MiB/s, 6 objects/s recovering 2024-09-13T14:25:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:14 smithi026 ceph-mon[99461]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:1880: done (4 left) 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:1881: done (3 left) 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:1882: done (2 left) 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:1883: done (1 left) 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:1884: done (0 left) 2024-09-13T14:25:14.921 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 175 2024-09-13T14:25:14.922 INFO:tasks.rados.rados.0.smithi026.stdout:1885: seq_num 613 ranges {3170304=671744} 2024-09-13T14:25:14.942 INFO:tasks.rados.rados.0.smithi026.stdout:1885: writing smithi02683976-38 from 3170304 to 3842048 tid 1 2024-09-13T14:25:14.942 INFO:tasks.rados.rados.0.smithi026.stdout:1886: snap_create 2024-09-13T14:25:14.946 INFO:tasks.rados.rados.0.smithi026.stdout:1885: finishing write tid 1 to smithi02683976-38 2024-09-13T14:25:14.947 INFO:tasks.rados.rados.0.smithi026.stdout:1885: finishing write tid 2 to smithi02683976-38 2024-09-13T14:25:14.947 INFO:tasks.rados.rados.0.smithi026.stdout:1885: finishing write tid 3 to smithi02683976-38 2024-09-13T14:25:14.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 841 (ObjNum 613 snap 175 seq_num 613) dirty exists 2024-09-13T14:25:14.948 INFO:tasks.rados.rados.0.smithi026.stdout:1885: left oid 38 (ObjNum 613 snap 175 seq_num 613) 2024-09-13T14:25:15.922 INFO:tasks.rados.rados.0.smithi026.stdout:1885: done (1 left) 2024-09-13T14:25:15.922 INFO:tasks.rados.rados.0.smithi026.stdout:1886: done (0 left) 2024-09-13T14:25:15.922 INFO:tasks.rados.rados.0.smithi026.stdout:1887: setattr oid 29 current snap is 176 2024-09-13T14:25:15.923 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 176 2024-09-13T14:25:15.923 INFO:tasks.rados.rados.0.smithi026.stdout:1888: seq_num 614 ranges {4513792=688128} 2024-09-13T14:25:15.952 INFO:tasks.rados.rados.0.smithi026.stdout:1888: writing smithi02683976-31 from 4513792 to 5201920 tid 1 2024-09-13T14:25:15.952 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 176 2024-09-13T14:25:15.953 INFO:tasks.rados.rados.0.smithi026.stdout:1889: seq_num 615 ranges {4456448=704512} 2024-09-13T14:25:15.980 INFO:tasks.rados.rados.0.smithi026.stdout:1889: writing smithi02683976-46 from 4456448 to 5160960 tid 1 2024-09-13T14:25:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 176 2024-09-13T14:25:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:1890: seq_num 616 ranges {7004160=720896} 2024-09-13T14:25:16.023 INFO:tasks.rados.rados.0.smithi026.stdout:1890: writing smithi02683976-14 from 7004160 to 7725056 tid 1 2024-09-13T14:25:16.023 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 176 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 962 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1888: finishing write tid 1 to smithi02683976-31 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1888: finishing write tid 2 to smithi02683976-31 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1888: finishing write tid 3 to smithi02683976-31 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 960 (ObjNum 614 snap 176 seq_num 614) dirty exists 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1888: left oid 31 (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1889: finishing write tid 1 to smithi02683976-46 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1889: finishing write tid 2 to smithi02683976-46 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1889: finishing write tid 3 to smithi02683976-46 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 466 (ObjNum 615 snap 176 seq_num 615) dirty exists 2024-09-13T14:25:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:1889: left oid 46 (ObjNum 615 snap 176 seq_num 615) 2024-09-13T14:25:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:1891: seq_num 617 ranges {0=737280} 2024-09-13T14:25:16.027 INFO:tasks.rados.rados.0.smithi026.stdout:1891: writing smithi02683976-12 from 0 to 737280 tid 1 2024-09-13T14:25:16.027 INFO:tasks.rados.rados.0.smithi026.stdout:1887: done (4 left) 2024-09-13T14:25:16.027 INFO:tasks.rados.rados.0.smithi026.stdout:1888: done (3 left) 2024-09-13T14:25:16.027 INFO:tasks.rados.rados.0.smithi026.stdout:1889: done (2 left) 2024-09-13T14:25:16.027 INFO:tasks.rados.rados.0.smithi026.stdout:1892: rmattr oid 43 current snap is 176 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:1890: finishing write tid 1 to smithi02683976-14 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:1890: finishing write tid 2 to smithi02683976-14 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:1893: setattr oid 33 current snap is 176 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:1890: finishing write tid 3 to smithi02683976-14 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 844 (ObjNum 616 snap 176 seq_num 616) dirty exists 2024-09-13T14:25:16.028 INFO:tasks.rados.rados.0.smithi026.stdout:1890: left oid 14 (ObjNum 616 snap 176 seq_num 616) 2024-09-13T14:25:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:1890: done (3 left) 2024-09-13T14:25:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:1894: rollback oid 4 current snap is 176 2024-09-13T14:25:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 175 2024-09-13T14:25:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 176 2024-09-13T14:25:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:1895: seq_num 618 ranges {303104=753664} 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:1895: writing smithi02683976-8 from 303104 to 1056768 tid 1 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:1896: rollback oid 28 current snap is 176 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 174 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:1891: finishing write tid 1 to smithi02683976-12 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 962 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:1894: finishing rollback tid 0 to smithi02683976-4 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 963 (ObjNum 596 snap 170 seq_num 596) dirty exists 2024-09-13T14:25:16.036 INFO:tasks.rados.rados.0.smithi026.stdout:1894: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:1892: done (5 left) 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:1893: done (4 left) 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:1897: copy_from oid 38 from oid 21 current snap is 176 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:1891: finishing write tid 2 to smithi02683976-12 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:1894: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:25:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 600 (ObjNum 604 snap 172 seq_num 604) dirty exists 2024-09-13T14:25:16.038 INFO:tasks.rados.rados.0.smithi026.stdout:1894: done (4 left) 2024-09-13T14:25:16.038 INFO:tasks.rados.rados.0.smithi026.stdout:1898: delete oid 3 current snap is 176 2024-09-13T14:25:16.038 INFO:tasks.rados.rados.0.smithi026.stdout:1891: finishing write tid 3 to smithi02683976-12 2024-09-13T14:25:16.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 945 (ObjNum 617 snap 176 seq_num 617) dirty exists 2024-09-13T14:25:16.038 INFO:tasks.rados.rados.0.smithi026.stdout:1891: left oid 12 (ObjNum 617 snap 176 seq_num 617) 2024-09-13T14:25:16.039 INFO:tasks.rados.rados.0.smithi026.stdout:1891: done (4 left) 2024-09-13T14:25:16.039 INFO:tasks.rados.rados.0.smithi026.stdout:1898: done (3 left) 2024-09-13T14:25:16.040 INFO:tasks.rados.rados.0.smithi026.stdout:1899: rmattr oid 27 current snap is 176 2024-09-13T14:25:16.040 INFO:tasks.rados.rados.0.smithi026.stdout:1899: done (3 left) 2024-09-13T14:25:16.040 INFO:tasks.rados.rados.0.smithi026.stdout:1900: copy_from oid 48 from oid 47 current snap is 176 2024-09-13T14:25:16.040 INFO:tasks.rados.rados.0.smithi026.stdout:1901: setattr oid 24 current snap is 176 2024-09-13T14:25:16.040 INFO:tasks.rados.rados.0.smithi026.stdout:1895: finishing write tid 1 to smithi02683976-8 2024-09-13T14:25:16.041 INFO:tasks.rados.rados.0.smithi026.stdout:1900: finishing copy_from to smithi02683976-48 2024-09-13T14:25:16.041 INFO:tasks.rados.rados.0.smithi026.stdout:1900: got expected ENOENT (src dne) 2024-09-13T14:25:16.041 INFO:tasks.rados.rados.0.smithi026.stdout:1895: finishing write tid 2 to smithi02683976-8 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1902: snap_create 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1896: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1900: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1896: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1896: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 824 (ObjNum 530 snap 148 seq_num 530) dirty exists 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1895: finishing write tid 3 to smithi02683976-8 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 588 (ObjNum 618 snap 176 seq_num 618) dirty exists 2024-09-13T14:25:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:1895: left oid 8 (ObjNum 618 snap 176 seq_num 618) 2024-09-13T14:25:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 637 (ObjNum 542 snap 156 seq_num 542) dirty exists 2024-09-13T14:25:16.059 INFO:tasks.rados.rados.0.smithi026.stdout:1897: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:25:16.061 INFO:tasks.rados.rados.0.smithi026.stdout:1897: finishing copy_from to smithi02683976-38 2024-09-13T14:25:16.061 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 920 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:25:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:15 smithi026 ceph-mon[97022]: pgmap v227: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-13T14:25:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:15 smithi026 ceph-mon[97022]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:25:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:15 smithi026 ceph-mon[99461]: pgmap v227: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-13T14:25:16.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:15 smithi026 ceph-mon[99461]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:25:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:15 smithi145 ceph-mon[89145]: pgmap v227: 121 pgs: 121 active+clean; 196 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.7 MiB/s wr, 16 op/s 2024-09-13T14:25:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:15 smithi145 ceph-mon[89145]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:25:16.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:16.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:16.934 INFO:tasks.rados.rados.0.smithi026.stdout:1895: done (5 left) 2024-09-13T14:25:16.934 INFO:tasks.rados.rados.0.smithi026.stdout:1896: done (4 left) 2024-09-13T14:25:16.934 INFO:tasks.rados.rados.0.smithi026.stdout:1897: done (3 left) 2024-09-13T14:25:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1900: done (2 left) 2024-09-13T14:25:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1901: done (1 left) 2024-09-13T14:25:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1902: done (0 left) 2024-09-13T14:25:16.935 INFO:tasks.rados.rados.0.smithi026.stdout:1903: delete oid 24 current snap is 177 2024-09-13T14:25:16.939 INFO:tasks.rados.rados.0.smithi026.stdout:1903: done (0 left) 2024-09-13T14:25:16.939 INFO:tasks.rados.rados.0.smithi026.stdout:1904: rmattr oid 14 current snap is 177 2024-09-13T14:25:16.939 INFO:tasks.rados.rados.0.smithi026.stdout:1904: done (0 left) 2024-09-13T14:25:16.939 INFO:tasks.rados.rados.0.smithi026.stdout:1905: delete oid 15 current snap is 177 2024-09-13T14:25:16.942 INFO:tasks.rados.rados.0.smithi026.stdout:1905: done (0 left) 2024-09-13T14:25:16.942 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 177 2024-09-13T14:25:16.942 INFO:tasks.rados.rados.0.smithi026.stdout:1906: seq_num 622 ranges {3694592=425984,4120576=401408} 2024-09-13T14:25:16.965 INFO:tasks.rados.rados.0.smithi026.stdout:1906: writing smithi02683976-4 from 3694592 to 4120576 tid 1 2024-09-13T14:25:16.967 INFO:tasks.rados.rados.0.smithi026.stdout:1906: writing smithi02683976-4 from 4120576 to 4521984 tid 2 2024-09-13T14:25:16.967 INFO:tasks.rados.rados.0.smithi026.stdout:1907: snap_create 2024-09-13T14:25:16.969 INFO:tasks.rados.rados.0.smithi026.stdout:1906: finishing write tid 1 to smithi02683976-4 2024-09-13T14:25:16.970 INFO:tasks.rados.rados.0.smithi026.stdout:1906: finishing write tid 2 to smithi02683976-4 2024-09-13T14:25:16.970 INFO:tasks.rados.rados.0.smithi026.stdout:1906: finishing write tid 3 to smithi02683976-4 2024-09-13T14:25:16.970 INFO:tasks.rados.rados.0.smithi026.stdout:1906: finishing write tid 4 to smithi02683976-4 2024-09-13T14:25:16.971 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 604 (ObjNum 622 snap 177 seq_num 622) dirty exists 2024-09-13T14:25:16.971 INFO:tasks.rados.rados.0.smithi026.stdout:1906: left oid 4 (ObjNum 622 snap 177 seq_num 622) 2024-09-13T14:25:17.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:16 smithi145 ceph-mon[89145]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:25:17.251 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:16 smithi026 ceph-mon[97022]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:25:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:16 smithi026 ceph-mon[99461]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:25:17.936 INFO:tasks.rados.rados.0.smithi026.stdout:1906: done (1 left) 2024-09-13T14:25:17.936 INFO:tasks.rados.rados.0.smithi026.stdout:1907: done (0 left) 2024-09-13T14:25:17.936 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 178 2024-09-13T14:25:17.936 INFO:tasks.rados.rados.0.smithi026.stdout:1908: seq_num 623 ranges {6356992=442368,6799360=401408} 2024-09-13T14:25:17.973 INFO:tasks.rados.rados.0.smithi026.stdout:1908: writing smithi02683976-45 from 6356992 to 6799360 tid 1 2024-09-13T14:25:17.975 INFO:tasks.rados.rados.0.smithi026.stdout:1908: writing smithi02683976-45 from 6799360 to 7200768 tid 2 2024-09-13T14:25:17.975 INFO:tasks.rados.rados.0.smithi026.stdout:1909: snap_remove snap 175 2024-09-13T14:25:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:17 smithi145 ceph-mon[89145]: pgmap v230: 121 pgs: 121 active+clean; 200 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1006 KiB/s wr, 17 op/s 2024-09-13T14:25:18.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:17 smithi145 ceph-mon[89145]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:25:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:17 smithi026 ceph-mon[97022]: pgmap v230: 121 pgs: 121 active+clean; 200 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1006 KiB/s wr, 17 op/s 2024-09-13T14:25:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:17 smithi026 ceph-mon[97022]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:25:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:17 smithi026 ceph-mon[99461]: pgmap v230: 121 pgs: 121 active+clean; 200 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1006 KiB/s wr, 17 op/s 2024-09-13T14:25:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:17 smithi026 ceph-mon[99461]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:25:18.922 INFO:tasks.rados.rados.0.smithi026.stdout:1909: done (1 left) 2024-09-13T14:25:18.922 INFO:tasks.rados.rados.0.smithi026.stdout:1910: copy_from oid 25 from oid 40 current snap is 178 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: finishing write tid 1 to smithi02683976-45 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: finishing write tid 2 to smithi02683976-45 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: finishing write tid 3 to smithi02683976-45 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: finishing write tid 4 to smithi02683976-45 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 816 (ObjNum 623 snap 178 seq_num 623) dirty exists 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: left oid 45 (ObjNum 623 snap 178 seq_num 623) 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:1908: done (1 left) 2024-09-13T14:25:18.923 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 178 2024-09-13T14:25:18.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: seq_num 624 ranges {270336=458752,729088=401408} 2024-09-13T14:25:18.927 INFO:tasks.rados.rados.0.smithi026.stdout:1911: writing smithi02683976-32 from 270336 to 729088 tid 1 2024-09-13T14:25:18.930 INFO:tasks.rados.rados.0.smithi026.stdout:1911: writing smithi02683976-32 from 729088 to 1130496 tid 2 2024-09-13T14:25:18.930 INFO:tasks.rados.rados.0.smithi026.stdout:1912: snap_remove snap 174 2024-09-13T14:25:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:18 smithi145 ceph-mon[89145]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:25:19.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:18 smithi145 ceph-mon[89145]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:25:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:18 smithi026 ceph-mon[97022]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:25:19.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:18 smithi026 ceph-mon[97022]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:25:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:18 smithi026 ceph-mon[99461]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:25:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:18 smithi026 ceph-mon[99461]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:25:19.923 INFO:tasks.rados.rados.0.smithi026.stdout:1912: done (2 left) 2024-09-13T14:25:19.923 INFO:tasks.rados.rados.0.smithi026.stdout:1913: delete oid 19 current snap is 178 2024-09-13T14:25:19.923 INFO:tasks.rados.rados.0.smithi026.stdout:1910: finishing copy_from to smithi02683976-25 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1910: got expected ENOENT (src dne) 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1910: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: finishing write tid 1 to smithi02683976-32 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: finishing write tid 2 to smithi02683976-32 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: finishing write tid 3 to smithi02683976-32 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: finishing write tid 4 to smithi02683976-32 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 923 (ObjNum 624 snap 178 seq_num 624) dirty exists 2024-09-13T14:25:19.924 INFO:tasks.rados.rados.0.smithi026.stdout:1911: left oid 32 (ObjNum 624 snap 178 seq_num 624) 2024-09-13T14:25:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1910: done (2 left) 2024-09-13T14:25:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1911: done (1 left) 2024-09-13T14:25:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1913: done (0 left) 2024-09-13T14:25:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 178 2024-09-13T14:25:19.931 INFO:tasks.rados.rados.0.smithi026.stdout:1914: seq_num 626 ranges {7397376=491520,7888896=401408} 2024-09-13T14:25:19.974 INFO:tasks.rados.rados.0.smithi026.stdout:1914: writing smithi02683976-23 from 7397376 to 7888896 tid 1 2024-09-13T14:25:19.977 INFO:tasks.rados.rados.0.smithi026.stdout:1914: writing smithi02683976-23 from 7888896 to 8290304 tid 2 2024-09-13T14:25:19.977 INFO:tasks.rados.rados.0.smithi026.stdout:1915: snap_create 2024-09-13T14:25:19.983 INFO:tasks.rados.rados.0.smithi026.stdout:1914: finishing write tid 1 to smithi02683976-23 2024-09-13T14:25:19.983 INFO:tasks.rados.rados.0.smithi026.stdout:1914: finishing write tid 2 to smithi02683976-23 2024-09-13T14:25:19.983 INFO:tasks.rados.rados.0.smithi026.stdout:1914: finishing write tid 3 to smithi02683976-23 2024-09-13T14:25:19.985 INFO:tasks.rados.rados.0.smithi026.stdout:1914: finishing write tid 4 to smithi02683976-23 2024-09-13T14:25:19.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 926 (ObjNum 626 snap 178 seq_num 626) dirty exists 2024-09-13T14:25:19.989 INFO:tasks.rados.rados.0.smithi026.stdout:1914: left oid 23 (ObjNum 626 snap 178 seq_num 626) 2024-09-13T14:25:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:19 smithi145 ceph-mon[89145]: pgmap v233: 121 pgs: 121 active+clean; 201 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-13T14:25:20.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:19 smithi145 ceph-mon[89145]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:25:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:19 smithi026 ceph-mon[97022]: pgmap v233: 121 pgs: 121 active+clean; 201 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-13T14:25:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:19 smithi026 ceph-mon[97022]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:25:20.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:19 smithi026 ceph-mon[99461]: pgmap v233: 121 pgs: 121 active+clean; 201 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-13T14:25:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:19 smithi026 ceph-mon[99461]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:25:20.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:20.955 INFO:tasks.rados.rados.0.smithi026.stdout:1914: done (1 left) 2024-09-13T14:25:20.955 INFO:tasks.rados.rados.0.smithi026.stdout:1915: done (0 left) 2024-09-13T14:25:20.955 INFO:tasks.rados.rados.0.smithi026.stdout:1916: read oid 33 snap -1 2024-09-13T14:25:20.955 INFO:tasks.rados.rados.0.smithi026.stdout:1916: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:25:20.955 INFO:tasks.rados.rados.0.smithi026.stdout:1917: delete oid 45 current snap is 179 2024-09-13T14:25:20.959 INFO:tasks.rados.rados.0.smithi026.stdout:1916: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1916: done (1 left) 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1917: done (0 left) 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1918: read oid 10 snap -1 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1918: expect (ObjNum 599 snap 171 seq_num 599) 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1919: read oid 44 snap -1 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1919: expect (ObjNum 559 snap 160 seq_num 559) 2024-09-13T14:25:20.985 INFO:tasks.rados.rados.0.smithi026.stdout:1920: setattr oid 35 current snap is 179 2024-09-13T14:25:20.987 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 179 2024-09-13T14:25:20.987 INFO:tasks.rados.rados.0.smithi026.stdout:1921: seq_num 628 ranges {3170304=524288,3694592=401408} 2024-09-13T14:25:21.007 INFO:tasks.rados.rados.0.smithi026.stdout:1921: writing smithi02683976-39 from 3170304 to 3694592 tid 1 2024-09-13T14:25:21.013 INFO:tasks.rados.rados.0.smithi026.stdout:1921: writing smithi02683976-39 from 3694592 to 4096000 tid 2 2024-09-13T14:25:21.013 INFO:tasks.rados.rados.0.smithi026.stdout:1922: snap_create 2024-09-13T14:25:21.013 INFO:tasks.rados.rados.0.smithi026.stdout:1918: expect (ObjNum 599 snap 171 seq_num 599) 2024-09-13T14:25:21.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 973 (ObjNum 177 snap 0 seq_num 0) dirty exists 2024-09-13T14:25:21.030 INFO:tasks.rados.rados.0.smithi026.stdout:1921: finishing write tid 1 to smithi02683976-39 2024-09-13T14:25:21.031 INFO:tasks.rados.rados.0.smithi026.stdout:1921: finishing write tid 2 to smithi02683976-39 2024-09-13T14:25:21.031 INFO:tasks.rados.rados.0.smithi026.stdout:1921: finishing write tid 3 to smithi02683976-39 2024-09-13T14:25:21.031 INFO:tasks.rados.rados.0.smithi026.stdout:1921: finishing write tid 4 to smithi02683976-39 2024-09-13T14:25:21.031 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 641 (ObjNum 628 snap 179 seq_num 628) dirty exists 2024-09-13T14:25:21.031 INFO:tasks.rados.rados.0.smithi026.stdout:1921: left oid 39 (ObjNum 628 snap 179 seq_num 628) 2024-09-13T14:25:21.042 INFO:tasks.rados.rados.0.smithi026.stdout:1919: expect (ObjNum 559 snap 160 seq_num 559) 2024-09-13T14:25:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:21] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1918: done (4 left) 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1919: done (3 left) 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1920: done (2 left) 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1921: done (1 left) 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1922: done (0 left) 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1923: read oid 17 snap -1 2024-09-13T14:25:21.956 INFO:tasks.rados.rados.0.smithi026.stdout:1923: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:21.957 INFO:tasks.rados.rados.0.smithi026.stdout:1924: snap_remove snap 179 2024-09-13T14:25:22.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[97022]: pgmap v236: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-13T14:25:22.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[97022]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:25:22.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:22.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[99461]: pgmap v236: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-13T14:25:22.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[99461]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:25:22.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:21 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:21 smithi145 ceph-mon[89145]: pgmap v236: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-13T14:25:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:21 smithi145 ceph-mon[89145]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:25:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:21 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:22.958 INFO:tasks.rados.rados.0.smithi026.stdout:1924: done (1 left) 2024-09-13T14:25:22.958 INFO:tasks.rados.rados.0.smithi026.stdout:1925: snap_remove snap 167 2024-09-13T14:25:23.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[99461]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:25:23.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[99461]: Upgrade: osd.3 is safe to restart 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[97022]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:22 smithi026 ceph-mon[97022]: Upgrade: osd.3 is safe to restart 2024-09-13T14:25:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:22 smithi145 ceph-mon[89145]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:25:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:25:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:22 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:25:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:22 smithi145 ceph-mon[89145]: Upgrade: osd.3 is safe to restart 2024-09-13T14:25:23.921 INFO:tasks.rados.rados.0.smithi026.stdout:1925: done (1 left) 2024-09-13T14:25:23.921 INFO:tasks.rados.rados.0.smithi026.stdout:1926: rmattr oid 32 current snap is 180 2024-09-13T14:25:23.921 INFO:tasks.rados.rados.0.smithi026.stdout:1926: done (1 left) 2024-09-13T14:25:23.921 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 180 2024-09-13T14:25:23.921 INFO:tasks.rados.rados.0.smithi026.stdout:1923: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:24.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: pgmap v239: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.2 MiB/s wr, 22 op/s 2024-09-13T14:25:24.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:25:24.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[97022]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: pgmap v239: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.2 MiB/s wr, 22 op/s 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:24.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:23 smithi026 ceph-mon[99461]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:25:24.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: pgmap v239: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.2 MiB/s wr, 22 op/s 2024-09-13T14:25:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:25:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:24.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:25:24.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:24.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:23 smithi145 ceph-mon[89145]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:25:24.746 INFO:tasks.rados.rados.0.smithi026.stdout:1927: seq_num 629 ranges {4096000=540672,4636672=401408} 2024-09-13T14:25:24.771 INFO:tasks.rados.rados.0.smithi026.stdout:1927: writing smithi02683976-39 from 4096000 to 4636672 tid 1 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:1927: writing smithi02683976-39 from 4636672 to 5038080 tid 2 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:1923: done (1 left) 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:1928: read oid 15 snap -1 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:1928: expect deleted 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 180 2024-09-13T14:25:24.774 INFO:tasks.rados.rados.0.smithi026.stdout:1929: seq_num 630 ranges {0=557056,557056=401408} 2024-09-13T14:25:24.777 INFO:tasks.rados.rados.0.smithi026.stdout:1929: writing smithi02683976-47 from 0 to 557056 tid 1 2024-09-13T14:25:24.779 INFO:tasks.rados.rados.0.smithi026.stdout:1929: writing smithi02683976-47 from 557056 to 958464 tid 2 2024-09-13T14:25:24.779 INFO:tasks.rados.rados.0.smithi026.stdout:1930: delete oid 42 current snap is 180 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:1927: finishing write tid 1 to smithi02683976-39 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:1927: finishing write tid 2 to smithi02683976-39 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:1927: finishing write tid 3 to smithi02683976-39 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:1927: finishing write tid 4 to smithi02683976-39 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 644 (ObjNum 629 snap 180 seq_num 629) dirty exists 2024-09-13T14:25:24.780 INFO:tasks.rados.rados.0.smithi026.stdout:1927: left oid 39 (ObjNum 629 snap 180 seq_num 629) 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1929: finishing write tid 1 to smithi02683976-47 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1927: done (3 left) 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1928: done (2 left) 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1930: done (1 left) 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1931: read oid 34 snap -1 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1931: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:24.782 INFO:tasks.rados.rados.0.smithi026.stdout:1932: delete oid 36 current snap is 180 2024-09-13T14:25:24.783 INFO:tasks.rados.rados.0.smithi026.stdout:1929: finishing write tid 2 to smithi02683976-47 2024-09-13T14:25:24.783 INFO:tasks.rados.rados.0.smithi026.stdout:1929: finishing write tid 3 to smithi02683976-47 2024-09-13T14:25:24.784 INFO:tasks.rados.rados.0.smithi026.stdout:1929: finishing write tid 4 to smithi02683976-47 2024-09-13T14:25:24.784 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 730 (ObjNum 630 snap 180 seq_num 630) dirty exists 2024-09-13T14:25:24.784 INFO:tasks.rados.rados.0.smithi026.stdout:1929: left oid 47 (ObjNum 630 snap 180 seq_num 630) 2024-09-13T14:25:24.786 INFO:tasks.rados.rados.0.smithi026.stdout:1929: done (2 left) 2024-09-13T14:25:24.786 INFO:tasks.rados.rados.0.smithi026.stdout:1932: done (1 left) 2024-09-13T14:25:24.786 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 180 2024-09-13T14:25:24.786 INFO:tasks.rados.rados.0.smithi026.stdout:1933: seq_num 633 ranges {0=606208,606208=401408} 2024-09-13T14:25:24.788 INFO:tasks.rados.rados.0.smithi026.stdout:1933: writing smithi02683976-42 from 0 to 606208 tid 1 2024-09-13T14:25:24.791 INFO:tasks.rados.rados.0.smithi026.stdout:1933: writing smithi02683976-42 from 606208 to 1007616 tid 2 2024-09-13T14:25:24.792 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 180 2024-09-13T14:25:24.792 INFO:tasks.rados.rados.0.smithi026.stdout:1934: seq_num 634 ranges {4513792=622592,5136384=401408} 2024-09-13T14:25:24.820 INFO:tasks.rados.rados.0.smithi026.stdout:1934: writing smithi02683976-2 from 4513792 to 5136384 tid 1 2024-09-13T14:25:24.822 INFO:tasks.rados.rados.0.smithi026.stdout:1934: writing smithi02683976-2 from 5136384 to 5537792 tid 2 2024-09-13T14:25:24.822 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 180 2024-09-13T14:25:24.822 INFO:tasks.rados.rados.0.smithi026.stdout:1933: finishing write tid 1 to smithi02683976-42 2024-09-13T14:25:24.822 INFO:tasks.rados.rados.0.smithi026.stdout:1935: seq_num 635 ranges {8290304=638976,8929280=401408} 2024-09-13T14:25:24.871 INFO:tasks.rados.rados.0.smithi026.stdout:1935: writing smithi02683976-23 from 8290304 to 8929280 tid 1 2024-09-13T14:25:24.874 INFO:tasks.rados.rados.0.smithi026.stdout:1935: writing smithi02683976-23 from 8929280 to 9330688 tid 2 2024-09-13T14:25:24.874 INFO:tasks.rados.rados.0.smithi026.stdout:1936: read oid 8 snap -1 2024-09-13T14:25:24.874 INFO:tasks.rados.rados.0.smithi026.stdout:1936: expect (ObjNum 618 snap 176 seq_num 618) 2024-09-13T14:25:24.875 INFO:tasks.rados.rados.0.smithi026.stdout:1937: copy_from oid 41 from oid 15 current snap is 180 2024-09-13T14:25:24.875 INFO:tasks.rados.rados.0.smithi026.stdout:1933: finishing write tid 2 to smithi02683976-42 2024-09-13T14:25:24.875 INFO:tasks.rados.rados.0.smithi026.stdout:1933: finishing write tid 3 to smithi02683976-42 2024-09-13T14:25:24.875 INFO:tasks.rados.rados.0.smithi026.stdout:1933: finishing write tid 4 to smithi02683976-42 2024-09-13T14:25:24.875 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 926 (ObjNum 633 snap 180 seq_num 633) dirty exists 2024-09-13T14:25:24.876 INFO:tasks.rados.rados.0.smithi026.stdout:1933: left oid 42 (ObjNum 633 snap 180 seq_num 633) 2024-09-13T14:25:24.876 INFO:tasks.rados.rados.0.smithi026.stdout:1931: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:25.224 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:24 smithi026 ceph-mon[99461]: Upgrade: Updating osd.3 2024-09-13T14:25:25.224 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:24 smithi026 ceph-mon[99461]: Deploying daemon osd.3 on smithi026 2024-09-13T14:25:25.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:24 smithi026 ceph-mon[97022]: Upgrade: Updating osd.3 2024-09-13T14:25:25.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:24 smithi026 ceph-mon[97022]: Deploying daemon osd.3 on smithi026 2024-09-13T14:25:25.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:24 smithi145 ceph-mon[89145]: Upgrade: Updating osd.3 2024-09-13T14:25:25.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:24 smithi145 ceph-mon[89145]: Deploying daemon osd.3 on smithi026 2024-09-13T14:25:25.693 INFO:tasks.rados.rados.0.smithi026.stdout:1934: finishing write tid 1 to smithi02683976-2 2024-09-13T14:25:25.693 INFO:tasks.rados.rados.0.smithi026.stdout:1934: finishing write tid 2 to smithi02683976-2 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1934: finishing write tid 3 to smithi02683976-2 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1934: finishing write tid 4 to smithi02683976-2 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 647 (ObjNum 634 snap 180 seq_num 634) dirty exists 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1934: left oid 2 (ObjNum 634 snap 180 seq_num 634) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1937: finishing copy_from to smithi02683976-41 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1937: got expected ENOENT (src dne) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1937: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1935: finishing write tid 1 to smithi02683976-23 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1931: done (5 left) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1933: done (4 left) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1934: done (3 left) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1937: done (2 left) 2024-09-13T14:25:25.694 INFO:tasks.rados.rados.0.smithi026.stdout:1938: read oid 32 snap -1 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:1938: expect (ObjNum 624 snap 178 seq_num 624) 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:1939: read oid 25 snap -1 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:1939: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 180 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:1935: finishing write tid 2 to smithi02683976-23 2024-09-13T14:25:25.695 INFO:tasks.rados.rados.0.smithi026.stdout:1940: seq_num 636 ranges {0=655360,655360=401408} 2024-09-13T14:25:25.697 INFO:tasks.rados.rados.0.smithi026.stdout:1940: writing smithi02683976-1 from 0 to 655360 tid 1 2024-09-13T14:25:25.700 INFO:tasks.rados.rados.0.smithi026.stdout:1940: writing smithi02683976-1 from 655360 to 1056768 tid 2 2024-09-13T14:25:25.700 INFO:tasks.rados.rados.0.smithi026.stdout:1941: read oid 20 snap -1 2024-09-13T14:25:25.701 INFO:tasks.rados.rados.0.smithi026.stdout:1941: expect (ObjNum 570 snap 163 seq_num 570) 2024-09-13T14:25:25.701 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 180 2024-09-13T14:25:25.701 INFO:tasks.rados.rados.0.smithi026.stdout:1942: seq_num 637 ranges {3776512=671744,4448256=401408} 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1942: writing smithi02683976-48 from 3776512 to 4448256 tid 1 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1942: writing smithi02683976-48 from 4448256 to 4849664 tid 2 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1943: rollback oid 13 current snap is 180 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 166 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1935: finishing write tid 3 to smithi02683976-23 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1935: finishing write tid 4 to smithi02683976-23 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 929 (ObjNum 635 snap 180 seq_num 635) dirty exists 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1935: left oid 23 (ObjNum 635 snap 180 seq_num 635) 2024-09-13T14:25:25.729 INFO:tasks.rados.rados.0.smithi026.stdout:1936: expect (ObjNum 618 snap 176 seq_num 618) 2024-09-13T14:25:25.762 INFO:tasks.rados.rados.0.smithi026.stdout:1938: expect (ObjNum 624 snap 178 seq_num 624) 2024-09-13T14:25:25.797 INFO:tasks.rados.rados.0.smithi026.stdout:1940: finishing write tid 1 to smithi02683976-1 2024-09-13T14:25:25.797 INFO:tasks.rados.rados.0.smithi026.stdout:1940: finishing write tid 2 to smithi02683976-1 2024-09-13T14:25:25.797 INFO:tasks.rados.rados.0.smithi026.stdout:1940: finishing write tid 3 to smithi02683976-1 2024-09-13T14:25:25.797 INFO:tasks.rados.rados.0.smithi026.stdout:1940: finishing write tid 4 to smithi02683976-1 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 981 (ObjNum 636 snap 180 seq_num 636) dirty exists 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1940: left oid 1 (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1935: done (7 left) 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1936: done (6 left) 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1938: done (5 left) 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1940: done (4 left) 2024-09-13T14:25:25.798 INFO:tasks.rados.rados.0.smithi026.stdout:1944: snap_remove snap 169 2024-09-13T14:25:25.972 INFO:tasks.rados.rados.0.smithi026.stdout:1944: done (4 left) 2024-09-13T14:25:25.972 INFO:tasks.rados.rados.0.smithi026.stdout:1945: rollback oid 2 current snap is 180 2024-09-13T14:25:25.973 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 173 2024-09-13T14:25:25.973 INFO:tasks.rados.rados.0.smithi026.stdout:1939: expect (ObjNum 378 snap 106 seq_num 378) 2024-09-13T14:25:26.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:25 smithi026 ceph-mon[99461]: pgmap v242: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-13T14:25:26.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:25 smithi026 ceph-mon[97022]: pgmap v242: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-13T14:25:26.179 INFO:tasks.rados.rados.0.smithi026.stdout:1941: expect (ObjNum 570 snap 163 seq_num 570) 2024-09-13T14:25:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:25 smithi145 ceph-mon[89145]: pgmap v242: 121 pgs: 121 active+clean; 209 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-13T14:25:26.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:26.338 INFO:tasks.rados.rados.0.smithi026.stdout:1942: finishing write tid 1 to smithi02683976-48 2024-09-13T14:25:26.338 INFO:tasks.rados.rados.0.smithi026.stdout:1942: finishing write tid 2 to smithi02683976-48 2024-09-13T14:25:26.338 INFO:tasks.rados.rados.0.smithi026.stdout:1942: finishing write tid 3 to smithi02683976-48 2024-09-13T14:25:26.338 INFO:tasks.rados.rados.0.smithi026.stdout:1942: finishing write tid 4 to smithi02683976-48 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 733 (ObjNum 637 snap 180 seq_num 637) dirty exists 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1942: left oid 48 (ObjNum 637 snap 180 seq_num 637) 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1943: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1943: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1943: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 966 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1945: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1945: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1945: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 650 (ObjNum 534 snap 149 seq_num 534) dirty exists 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1939: done (4 left) 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1941: done (3 left) 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1942: done (2 left) 2024-09-13T14:25:26.339 INFO:tasks.rados.rados.0.smithi026.stdout:1943: done (1 left) 2024-09-13T14:25:26.340 INFO:tasks.rados.rados.0.smithi026.stdout:1945: done (0 left) 2024-09-13T14:25:26.340 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 180 2024-09-13T14:25:26.340 INFO:tasks.rados.rados.0.smithi026.stdout:1946: seq_num 638 ranges {2760704=688128,3448832=401408} 2024-09-13T14:25:26.357 INFO:tasks.rados.rados.0.smithi026.stdout:1946: writing smithi02683976-9 from 2760704 to 3448832 tid 1 2024-09-13T14:25:26.359 INFO:tasks.rados.rados.0.smithi026.stdout:1946: writing smithi02683976-9 from 3448832 to 3850240 tid 2 2024-09-13T14:25:26.359 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 180 2024-09-13T14:25:26.359 INFO:tasks.rados.rados.0.smithi026.stdout:1947: seq_num 639 ranges {5038080=704512,5742592=401408} 2024-09-13T14:25:26.390 INFO:tasks.rados.rados.0.smithi026.stdout:1947: writing smithi02683976-39 from 5038080 to 5742592 tid 1 2024-09-13T14:25:26.393 INFO:tasks.rados.rados.0.smithi026.stdout:1947: writing smithi02683976-39 from 5742592 to 6144000 tid 2 2024-09-13T14:25:26.393 INFO:tasks.rados.rados.0.smithi026.stdout:1948: snap_create 2024-09-13T14:25:26.393 INFO:tasks.rados.rados.0.smithi026.stdout:1946: finishing write tid 1 to smithi02683976-9 2024-09-13T14:25:26.393 INFO:tasks.rados.rados.0.smithi026.stdout:1946: finishing write tid 2 to smithi02683976-9 2024-09-13T14:25:26.393 INFO:tasks.rados.rados.0.smithi026.stdout:1946: finishing write tid 3 to smithi02683976-9 2024-09-13T14:25:26.394 INFO:tasks.rados.rados.0.smithi026.stdout:1946: finishing write tid 4 to smithi02683976-9 2024-09-13T14:25:26.394 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 984 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:25:26.394 INFO:tasks.rados.rados.0.smithi026.stdout:1946: left oid 9 (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:25:26.395 INFO:tasks.rados.rados.0.smithi026.stdout:1947: finishing write tid 1 to smithi02683976-39 2024-09-13T14:25:26.395 INFO:tasks.rados.rados.0.smithi026.stdout:1947: finishing write tid 2 to smithi02683976-39 2024-09-13T14:25:26.395 INFO:tasks.rados.rados.0.smithi026.stdout:1947: finishing write tid 3 to smithi02683976-39 2024-09-13T14:25:26.396 INFO:tasks.rados.rados.0.smithi026.stdout:1947: finishing write tid 4 to smithi02683976-39 2024-09-13T14:25:26.396 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 653 (ObjNum 639 snap 180 seq_num 639) dirty exists 2024-09-13T14:25:26.396 INFO:tasks.rados.rados.0.smithi026.stdout:1947: left oid 39 (ObjNum 639 snap 180 seq_num 639) 2024-09-13T14:25:26.978 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:26 smithi026 systemd[1]: Stopping Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:25:27.163 INFO:tasks.rados.rados.0.smithi026.stdout:1946: done (2 left) 2024-09-13T14:25:27.164 INFO:tasks.rados.rados.0.smithi026.stdout:1947: done (1 left) 2024-09-13T14:25:27.164 INFO:tasks.rados.rados.0.smithi026.stdout:1948: done (0 left) 2024-09-13T14:25:27.164 INFO:tasks.rados.rados.0.smithi026.stdout:1949: delete oid 50 current snap is 181 2024-09-13T14:25:27.172 INFO:tasks.rados.rados.0.smithi026.stdout:1949: done (0 left) 2024-09-13T14:25:27.172 INFO:tasks.rados.rados.0.smithi026.stdout:1950: rmattr oid 47 current snap is 181 2024-09-13T14:25:27.172 INFO:tasks.rados.rados.0.smithi026.stdout:1950: done (0 left) 2024-09-13T14:25:27.172 INFO:tasks.rados.rados.0.smithi026.stdout:1951: delete oid 50 current snap is 181 2024-09-13T14:25:27.176 INFO:tasks.rados.rados.0.smithi026.stdout:1951: done (0 left) 2024-09-13T14:25:27.176 INFO:tasks.rados.rados.0.smithi026.stdout:1952: snap_remove snap 180 2024-09-13T14:25:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:26 smithi145 ceph-mon[89145]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:25:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:27.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:27.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[99461]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:25:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[97022]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:25:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:27.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[61949]: 2024-09-13T14:25:27.112+0000 7fe59d3bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:25:27.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[61949]: 2024-09-13T14:25:27.112+0000 7fe59d3bd640 -1 osd.3 430 *** Got signal Terminated *** 2024-09-13T14:25:27.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[61949]: 2024-09-13T14:25:27.112+0000 7fe59d3bd640 -1 osd.3 430 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:25:27.971 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:27 smithi026 podman[119842]: 2024-09-13 14:25:27.705811587 +0000 UTC m=+0.688604401 container died 8e148473b9fc21e37c45e1120d712441912252a9ac07a80a4edd638823c8b04c (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:25:28.167 INFO:tasks.rados.rados.0.smithi026.stdout:1952: done (0 left) 2024-09-13T14:25:28.167 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 181 2024-09-13T14:25:28.168 INFO:tasks.rados.rados.0.smithi026.stdout:1953: seq_num 642 ranges {9330688=761856,10092544=401408} 2024-09-13T14:25:28.223 INFO:tasks.rados.rados.0.smithi026.stdout:1953: writing smithi02683976-23 from 9330688 to 10092544 tid 1 2024-09-13T14:25:28.225 INFO:tasks.rados.rados.0.smithi026.stdout:1953: writing smithi02683976-23 from 10092544 to 10493952 tid 2 2024-09-13T14:25:28.225 INFO:tasks.rados.rados.0.smithi026.stdout:1954: read oid 15 snap -1 2024-09-13T14:25:28.225 INFO:tasks.rados.rados.0.smithi026.stdout:1954: expect deleted 2024-09-13T14:25:28.226 INFO:tasks.rados.rados.0.smithi026.stdout:1955: setattr oid 36 current snap is 181 2024-09-13T14:25:28.227 INFO:tasks.rados.rados.0.smithi026.stdout:1954: done (2 left) 2024-09-13T14:25:28.227 INFO:tasks.rados.rados.0.smithi026.stdout:1956: snap_create 2024-09-13T14:25:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:1953: finishing write tid 1 to smithi02683976-23 2024-09-13T14:25:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:1953: finishing write tid 2 to smithi02683976-23 2024-09-13T14:25:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:1953: finishing write tid 3 to smithi02683976-23 2024-09-13T14:25:28.229 INFO:tasks.rados.rados.0.smithi026.stdout:1953: finishing write tid 4 to smithi02683976-23 2024-09-13T14:25:28.230 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 932 (ObjNum 642 snap 181 seq_num 642) dirty exists 2024-09-13T14:25:28.230 INFO:tasks.rados.rados.0.smithi026.stdout:1953: left oid 23 (ObjNum 642 snap 181 seq_num 642) 2024-09-13T14:25:28.230 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 986 (ObjNum 129 snap 0 seq_num 519230312) dirty exists 2024-09-13T14:25:28.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[99461]: pgmap v244: 121 pgs: 121 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-13T14:25:28.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[99461]: osd.3 marked itself down and dead 2024-09-13T14:25:28.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:25:28.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[99461]: osdmap e431: 8 total, 7 up, 8 in 2024-09-13T14:25:28.355 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119842]: 2024-09-13 14:25:28.060792278 +0000 UTC m=+1.043585090 container cleanup 8e148473b9fc21e37c45e1120d712441912252a9ac07a80a4edd638823c8b04c (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T14:25:28.355 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 bash[119842]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:25:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[97022]: pgmap v244: 121 pgs: 121 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-13T14:25:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[97022]: osd.3 marked itself down and dead 2024-09-13T14:25:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:25:28.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:27 smithi026 ceph-mon[97022]: osdmap e431: 8 total, 7 up, 8 in 2024-09-13T14:25:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:27 smithi145 ceph-mon[89145]: pgmap v244: 121 pgs: 121 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-13T14:25:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:27 smithi145 ceph-mon[89145]: osd.3 marked itself down and dead 2024-09-13T14:25:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:27 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:25:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:27 smithi145 ceph-mon[89145]: osdmap e431: 8 total, 7 up, 8 in 2024-09-13T14:25:28.665 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119863]: 2024-09-13 14:25:28.353948196 +0000 UTC m=+0.650961493 container remove 8e148473b9fc21e37c45e1120d712441912252a9ac07a80a4edd638823c8b04c (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:25:28.923 INFO:tasks.rados.rados.0.smithi026.stdout:1953: done (2 left) 2024-09-13T14:25:28.924 INFO:tasks.rados.rados.0.smithi026.stdout:1955: done (1 left) 2024-09-13T14:25:28.924 INFO:tasks.rados.rados.0.smithi026.stdout:1956: done (0 left) 2024-09-13T14:25:28.924 INFO:tasks.rados.rados.0.smithi026.stdout:1957: delete oid 2 current snap is 182 2024-09-13T14:25:28.925 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119956]: 2024-09-13 14:25:28.571903044 +0000 UTC m=+0.016519591 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:25:28.925 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119956]: 2024-09-13 14:25:28.688406737 +0000 UTC m=+0.133023281 container create 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1957: done (0 left) 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1958: copy_from oid 29 from oid 30 current snap is 182 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1959: read oid 1 snap 181 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1959: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1960: rollback oid 17 current snap is 182 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 176 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 182 2024-09-13T14:25:28.929 INFO:tasks.rados.rados.0.smithi026.stdout:1961: seq_num 644 ranges {0=794624,794624=401408} 2024-09-13T14:25:28.934 INFO:tasks.rados.rados.0.smithi026.stdout:1961: writing smithi02683976-49 from 0 to 794624 tid 1 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1961: writing smithi02683976-49 from 794624 to 1196032 tid 2 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1962: setattr oid 5 current snap is 182 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1958: finishing copy_from to smithi02683976-29 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1958: got expected ENOENT (src dne) 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1958: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1960: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:25:28.937 INFO:tasks.rados.rados.0.smithi026.stdout:1960: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:25:28.938 INFO:tasks.rados.rados.0.smithi026.stdout:1960: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:25:28.938 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 810 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:28.938 INFO:tasks.rados.rados.0.smithi026.stdout:1959: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: finishing write tid 1 to smithi02683976-49 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: finishing write tid 2 to smithi02683976-49 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: finishing write tid 3 to smithi02683976-49 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: finishing write tid 4 to smithi02683976-49 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 827 (ObjNum 644 snap 182 seq_num 644) dirty exists 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: left oid 49 (ObjNum 644 snap 182 seq_num 644) 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1958: done (4 left) 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1959: done (3 left) 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1960: done (2 left) 2024-09-13T14:25:28.966 INFO:tasks.rados.rados.0.smithi026.stdout:1961: done (1 left) 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:1963: read oid 41 snap 173 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:1963: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:1964: rmattr oid 9 current snap is 182 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:1964: done (2 left) 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 182 2024-09-13T14:25:28.967 INFO:tasks.rados.rados.0.smithi026.stdout:1965: seq_num 645 ranges {4595712=409600,5005312=778240,5783552=24576} 2024-09-13T14:25:28.993 INFO:tasks.rados.rados.0.smithi026.stdout:1965: writing smithi02683976-16 from 4595712 to 5005312 tid 1 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1965: writing smithi02683976-16 from 5005312 to 5783552 tid 2 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1965: writing smithi02683976-16 from 5783552 to 5808128 tid 3 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1966: rollback oid 24 current snap is 182 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 178 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 988 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1962: done (3 left) 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1967: copy_from oid 11 from oid 8 current snap is 182 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1968: rollback oid 23 current snap is 182 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 23 to 166 2024-09-13T14:25:28.998 INFO:tasks.rados.rados.0.smithi026.stdout:1969: setattr oid 22 current snap is 182 2024-09-13T14:25:28.999 INFO:tasks.rados.rados.0.smithi026.stdout:1970: read oid 29 snap -1 2024-09-13T14:25:28.999 INFO:tasks.rados.rados.0.smithi026.stdout:1970: expect (ObjNum 607 snap 173 seq_num 607) 2024-09-13T14:25:28.999 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 182 2024-09-13T14:25:29.000 INFO:tasks.rados.rados.0.smithi026.stdout:1971: seq_num 646 ranges {1130496=425984,1556480=655360,2211840=147456} 2024-09-13T14:25:29.009 INFO:tasks.rados.rados.0.smithi026.stdout:1971: writing smithi02683976-32 from 1130496 to 1556480 tid 1 2024-09-13T14:25:29.013 INFO:tasks.rados.rados.0.smithi026.stdout:1971: writing smithi02683976-32 from 1556480 to 2211840 tid 2 2024-09-13T14:25:29.015 INFO:tasks.rados.rados.0.smithi026.stdout:1971: writing smithi02683976-32 from 2211840 to 2359296 tid 3 2024-09-13T14:25:29.015 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 182 2024-09-13T14:25:29.015 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 946 (ObjNum 590 snap 169 seq_num 590) dirty exists 2024-09-13T14:25:29.015 INFO:tasks.rados.rados.0.smithi026.stdout:1968: finishing rollback tid 0 to smithi02683976-23 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1966: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 638 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1965: finishing write tid 1 to smithi02683976-16 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1968: finishing rollback tid 1 to smithi02683976-23 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1968: finishing rollback tid 2 to smithi02683976-23 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 935 (ObjNum 292 snap 80 seq_num 292) dirty exists 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1965: finishing write tid 2 to smithi02683976-16 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1965: finishing write tid 3 to smithi02683976-16 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1965: finishing write tid 4 to smithi02683976-16 2024-09-13T14:25:29.016 INFO:tasks.rados.rados.0.smithi026.stdout:1972: seq_num 647 ranges {4243456=442368,4685824=589824,5275648=212992} 2024-09-13T14:25:29.041 INFO:tasks.rados.rados.0.smithi026.stdout:1972: writing smithi02683976-25 from 4243456 to 4685824 tid 1 2024-09-13T14:25:29.044 INFO:tasks.rados.rados.0.smithi026.stdout:1972: writing smithi02683976-25 from 4685824 to 5275648 tid 2 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1972: writing smithi02683976-25 from 5275648 to 5488640 tid 3 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1966: done (8 left) 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1968: done (7 left) 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1969: done (6 left) 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1973: read oid 33 snap 173 2024-09-13T14:25:29.046 INFO:tasks.rados.rados.0.smithi026.stdout:1973: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:25:29.047 INFO:tasks.rados.rados.0.smithi026.stdout:1974: rollback oid 48 current snap is 182 2024-09-13T14:25:29.047 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 173 2024-09-13T14:25:29.047 INFO:tasks.rados.rados.0.smithi026.stdout:1963: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:25:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:29 smithi026 ceph-mon[99461]: osdmap e432: 8 total, 7 up, 8 in 2024-09-13T14:25:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:29 smithi026 ceph-mon[99461]: osdmap e433: 8 total, 7 up, 8 in 2024-09-13T14:25:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:29 smithi026 ceph-mon[97022]: osdmap e432: 8 total, 7 up, 8 in 2024-09-13T14:25:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:29 smithi026 ceph-mon[97022]: osdmap e433: 8 total, 7 up, 8 in 2024-09-13T14:25:29.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119956]: 2024-09-13 14:25:28.924603777 +0000 UTC m=+0.369220325 container init 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:25:29.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:28 smithi026 podman[119956]: 2024-09-13 14:25:28.933611174 +0000 UTC m=+0.378227722 container start 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:25:29.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 podman[119956]: 2024-09-13 14:25:29.032524753 +0000 UTC m=+0.477141299 container attach 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:25:29.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 podman[119956]: 2024-09-13 14:25:29.153134041 +0000 UTC m=+0.597750586 container died 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T14:25:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:29 smithi145 ceph-mon[89145]: osdmap e432: 8 total, 7 up, 8 in 2024-09-13T14:25:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:29 smithi145 ceph-mon[89145]: osdmap e433: 8 total, 7 up, 8 in 2024-09-13T14:25:29.873 INFO:tasks.rados.rados.0.smithi026.stdout:1967: finishing copy_from racing read to smithi02683976-11 2024-09-13T14:25:29.873 INFO:tasks.rados.rados.0.smithi026.stdout:1971: finishing write tid 1 to smithi02683976-32 2024-09-13T14:25:29.873 INFO:tasks.rados.rados.0.smithi026.stdout:1970: expect (ObjNum 607 snap 173 seq_num 607) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1965: finishing write tid 5 to smithi02683976-16 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 970 (ObjNum 645 snap 182 seq_num 645) dirty exists 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1965: left oid 16 (ObjNum 645 snap 182 seq_num 645) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1967: finishing copy_from to smithi02683976-11 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 588 (ObjNum 618 snap 176 seq_num 618) dirty exists 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1971: finishing write tid 2 to smithi02683976-32 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1971: finishing write tid 3 to smithi02683976-32 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1963: done (7 left) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1965: done (6 left) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1967: done (5 left) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:1970: done (4 left) 2024-09-13T14:25:29.892 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 182 2024-09-13T14:25:29.893 INFO:tasks.rados.rados.0.smithi026.stdout:1971: finishing write tid 4 to smithi02683976-32 2024-09-13T14:25:29.893 INFO:tasks.rados.rados.0.smithi026.stdout:1975: seq_num 648 ranges {5169152=458752,5627904=475136,6103040=327680} 2024-09-13T14:25:29.922 INFO:tasks.rados.rados.0.smithi026.stdout:1975: writing smithi02683976-22 from 5169152 to 5627904 tid 1 2024-09-13T14:25:29.925 INFO:tasks.rados.rados.0.smithi026.stdout:1975: writing smithi02683976-22 from 5627904 to 6103040 tid 2 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1975: writing smithi02683976-22 from 6103040 to 6430720 tid 3 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1976: delete oid 7 current snap is 182 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1971: finishing write tid 5 to smithi02683976-32 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 939 (ObjNum 646 snap 182 seq_num 646) dirty exists 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1971: left oid 32 (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1972: finishing write tid 1 to smithi02683976-25 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1972: finishing write tid 2 to smithi02683976-25 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1972: finishing write tid 3 to smithi02683976-25 2024-09-13T14:25:29.927 INFO:tasks.rados.rados.0.smithi026.stdout:1972: finishing write tid 4 to smithi02683976-25 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1974: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1974: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1972: finishing write tid 5 to smithi02683976-25 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 658 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1972: left oid 25 (ObjNum 647 snap 182 seq_num 647) 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1974: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 736 (ObjNum 609 snap 173 seq_num 609) dirty exists 2024-09-13T14:25:29.928 INFO:tasks.rados.rados.0.smithi026.stdout:1973: expect (ObjNum 480 snap 131 seq_num 480) 2024-09-13T14:25:29.950 INFO:tasks.rados.rados.0.smithi026.stdout:1975: finishing write tid 1 to smithi02683976-22 2024-09-13T14:25:29.950 INFO:tasks.rados.rados.0.smithi026.stdout:1975: finishing write tid 2 to smithi02683976-22 2024-09-13T14:25:29.950 INFO:tasks.rados.rados.0.smithi026.stdout:1975: finishing write tid 3 to smithi02683976-22 2024-09-13T14:25:29.950 INFO:tasks.rados.rados.0.smithi026.stdout:1975: finishing write tid 4 to smithi02683976-22 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1975: finishing write tid 5 to smithi02683976-22 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 950 (ObjNum 648 snap 182 seq_num 648) dirty exists 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1975: left oid 22 (ObjNum 648 snap 182 seq_num 648) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1971: done (5 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1972: done (4 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1973: done (3 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1974: done (2 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1975: done (1 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1976: done (0 left) 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1977: rollback oid 17 current snap is 182 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 178 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1978: read oid 28 snap 177 2024-09-13T14:25:29.951 INFO:tasks.rados.rados.0.smithi026.stdout:1978: expect (ObjNum 530 snap 148 seq_num 530) 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1979: read oid 32 snap -1 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1979: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1980: copy_from oid 22 from oid 12 current snap is 182 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1981: setattr oid 30 current snap is 182 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1982: read oid 27 snap -1 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1982: expect deleted 2024-09-13T14:25:29.952 INFO:tasks.rados.rados.0.smithi026.stdout:1983: read oid 40 snap -1 2024-09-13T14:25:29.953 INFO:tasks.rados.rados.0.smithi026.stdout:1983: expect deleted 2024-09-13T14:25:29.953 INFO:tasks.rados.rados.0.smithi026.stdout:1984: snap_create 2024-09-13T14:25:29.953 INFO:tasks.rados.rados.0.smithi026.stdout:1977: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:25:29.953 INFO:tasks.rados.rados.0.smithi026.stdout:1977: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:25:29.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 605 (ObjNum 33 snap 0 seq_num 519367904) dirty exists 2024-09-13T14:25:29.954 INFO:tasks.rados.rados.0.smithi026.stdout:1977: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:25:29.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 813 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:29.957 INFO:tasks.rados.rados.0.smithi026.stdout:1980: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:25:29.958 INFO:tasks.rados.rados.0.smithi026.stdout:1980: finishing copy_from to smithi02683976-22 2024-09-13T14:25:29.958 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 945 (ObjNum 617 snap 176 seq_num 617) dirty exists 2024-09-13T14:25:29.961 INFO:tasks.rados.rados.0.smithi026.stdout:1979: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:25:30.048 INFO:tasks.rados.rados.0.smithi026.stdout:1978: expect (ObjNum 530 snap 148 seq_num 530) 2024-09-13T14:25:30.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 podman[120000]: 2024-09-13 14:25:29.795007929 +0000 UTC m=+0.635634536 container remove 68c4843c3d1f60a5957a4cde8e6c319018f8a431cd654c166178cd31410c43f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T14:25:30.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:25:30.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 systemd[1]: Stopped Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:25:30.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:29 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service: Consumed 10.046s CPU time. 2024-09-13T14:25:30.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 systemd[1]: Starting Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1977: done (7 left) 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1978: done (6 left) 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1979: done (5 left) 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1980: done (4 left) 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1981: done (3 left) 2024-09-13T14:25:30.215 INFO:tasks.rados.rados.0.smithi026.stdout:1982: done (2 left) 2024-09-13T14:25:30.216 INFO:tasks.rados.rados.0.smithi026.stdout:1983: done (1 left) 2024-09-13T14:25:30.216 INFO:tasks.rados.rados.0.smithi026.stdout:1984: done (0 left) 2024-09-13T14:25:30.216 INFO:tasks.rados.rados.0.smithi026.stdout:1985: copy_from oid 43 from oid 29 current snap is 183 2024-09-13T14:25:30.216 INFO:tasks.rados.rados.0.smithi026.stdout:1986: delete oid 37 current snap is 183 2024-09-13T14:25:30.219 INFO:tasks.rados.rados.0.smithi026.stdout:1986: done (1 left) 2024-09-13T14:25:30.219 INFO:tasks.rados.rados.0.smithi026.stdout:1987: read oid 5 snap -1 2024-09-13T14:25:30.219 INFO:tasks.rados.rados.0.smithi026.stdout:1987: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:25:30.220 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 183 2024-09-13T14:25:30.220 INFO:tasks.rados.rados.0.smithi026.stdout:1988: seq_num 651 ranges {0=507904,507904=532480,1040384=270336} 2024-09-13T14:25:30.222 INFO:tasks.rados.rados.0.smithi026.stdout:1988: writing smithi02683976-15 from 0 to 507904 tid 1 2024-09-13T14:25:30.229 INFO:tasks.rados.rados.0.smithi026.stdout:1988: writing smithi02683976-15 from 507904 to 1040384 tid 2 2024-09-13T14:25:30.229 INFO:tasks.rados.rados.0.smithi026.stdout:1988: writing smithi02683976-15 from 1040384 to 1310720 tid 3 2024-09-13T14:25:30.230 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 183 2024-09-13T14:25:30.230 INFO:tasks.rados.rados.0.smithi026.stdout:1985: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:25:30.230 INFO:tasks.rados.rados.0.smithi026.stdout:1985: finishing copy_from to smithi02683976-43 2024-09-13T14:25:30.230 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 962 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:25:30.230 INFO:tasks.rados.rados.0.smithi026.stdout:1989: seq_num 652 ranges {0=524288,524288=409600,933888=393216} 2024-09-13T14:25:30.232 INFO:tasks.rados.rados.0.smithi026.stdout:1989: writing smithi02683976-40 from 0 to 524288 tid 1 2024-09-13T14:25:30.234 INFO:tasks.rados.rados.0.smithi026.stdout:1989: writing smithi02683976-40 from 524288 to 933888 tid 2 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1989: writing smithi02683976-40 from 933888 to 1327104 tid 3 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1985: done (3 left) 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1990: delete oid 11 current snap is 183 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: finishing write tid 1 to smithi02683976-15 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: finishing write tid 2 to smithi02683976-15 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: finishing write tid 3 to smithi02683976-15 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: finishing write tid 4 to smithi02683976-15 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: finishing write tid 5 to smithi02683976-15 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 609 (ObjNum 651 snap 183 seq_num 651) dirty exists 2024-09-13T14:25:30.238 INFO:tasks.rados.rados.0.smithi026.stdout:1988: left oid 15 (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:25:30.239 INFO:tasks.rados.rados.0.smithi026.stdout:1989: finishing write tid 1 to smithi02683976-40 2024-09-13T14:25:30.243 INFO:tasks.rados.rados.0.smithi026.stdout:1987: expect (ObjNum 292 snap 80 seq_num 292) 2024-09-13T14:25:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:30 smithi145 ceph-mon[89145]: pgmap v247: 121 pgs: 17 stale+active+clean, 104 active+clean; 215 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-13T14:25:30.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:30.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-mon[99461]: pgmap v247: 121 pgs: 17 stale+active+clean, 104 active+clean; 215 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-13T14:25:30.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-mon[97022]: pgmap v247: 121 pgs: 17 stale+active+clean, 104 active+clean; 215 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-13T14:25:30.516 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 podman[120103]: 2024-09-13 14:25:30.150685114 +0000 UTC m=+0.018304283 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:25:30.517 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 podman[120103]: 2024-09-13 14:25:30.279965255 +0000 UTC m=+0.147584415 container create 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1989: finishing write tid 2 to smithi02683976-40 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1989: finishing write tid 3 to smithi02683976-40 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1987: done (3 left) 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1988: done (2 left) 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1990: done (1 left) 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1991: read oid 39 snap -1 2024-09-13T14:25:30.677 INFO:tasks.rados.rados.0.smithi026.stdout:1991: expect (ObjNum 639 snap 180 seq_num 639) 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1989: finishing write tid 4 to smithi02683976-40 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1989: finishing write tid 5 to smithi02683976-40 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 831 (ObjNum 652 snap 183 seq_num 652) dirty exists 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1989: left oid 40 (ObjNum 652 snap 183 seq_num 652) 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1992: read oid 31 snap -1 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1992: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1989: done (2 left) 2024-09-13T14:25:30.678 INFO:tasks.rados.rados.0.smithi026.stdout:1993: snap_create 2024-09-13T14:25:30.695 INFO:tasks.rados.rados.0.smithi026.stdout:1991: expect (ObjNum 639 snap 180 seq_num 639) 2024-09-13T14:25:30.916 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 podman[120103]: 2024-09-13 14:25:30.515578598 +0000 UTC m=+0.383197751 container init 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-09-13T14:25:30.916 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 podman[120103]: 2024-09-13 14:25:30.520674036 +0000 UTC m=+0.388293198 container start 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:25:30.916 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 podman[120103]: 2024-09-13 14:25:30.649396443 +0000 UTC m=+0.517015606 container attach 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:25:30.916 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:25:30.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:25:30.918 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:25:30.918 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:25:30.918 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:30 smithi026 bash[120103]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:25:31.124 INFO:tasks.rados.rados.0.smithi026.stdout:1992: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:25:31.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-mon[97022]: osdmap e434: 8 total, 7 up, 8 in 2024-09-13T14:25:31.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-mon[99461]: osdmap e434: 8 total, 7 up, 8 in 2024-09-13T14:25:31.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:25:31.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 bash[120103]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:25:31.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:25:31.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 bash[120103]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:25:31.352 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 bash[120103]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 bash[120103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate[120118]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 bash[120103]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:25:31.353 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 podman[120103]: 2024-09-13 14:25:31.075670884 +0000 UTC m=+0.943290045 container died 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:25:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:31 smithi145 ceph-mon[89145]: osdmap e434: 8 total, 7 up, 8 in 2024-09-13T14:25:31.455 INFO:tasks.rados.rados.0.smithi026.stdout:1991: done (2 left) 2024-09-13T14:25:31.456 INFO:tasks.rados.rados.0.smithi026.stdout:1992: done (1 left) 2024-09-13T14:25:31.456 INFO:tasks.rados.rados.0.smithi026.stdout:1993: done (0 left) 2024-09-13T14:25:31.456 INFO:tasks.rados.rados.0.smithi026.stdout:1994: setattr oid 45 current snap is 184 2024-09-13T14:25:31.456 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 184 2024-09-13T14:25:31.456 INFO:tasks.rados.rados.0.smithi026.stdout:1995: seq_num 654 ranges {4521984=557056,5079040=630784,5709824=172032} 2024-09-13T14:25:31.484 INFO:tasks.rados.rados.0.smithi026.stdout:1995: writing smithi02683976-4 from 4521984 to 5079040 tid 1 2024-09-13T14:25:31.487 INFO:tasks.rados.rados.0.smithi026.stdout:1995: writing smithi02683976-4 from 5079040 to 5709824 tid 2 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:1995: writing smithi02683976-4 from 5709824 to 5881856 tid 3 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:1996: rollback oid 26 current snap is 184 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 182 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 818 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:1995: finishing write tid 1 to smithi02683976-4 2024-09-13T14:25:31.489 INFO:tasks.rados.rados.0.smithi026.stdout:1994: done (2 left) 2024-09-13T14:25:31.490 INFO:tasks.rados.rados.0.smithi026.stdout:1997: copy_from oid 14 from oid 24 current snap is 184 2024-09-13T14:25:31.490 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 184 2024-09-13T14:25:31.490 INFO:tasks.rados.rados.0.smithi026.stdout:1998: seq_num 655 ranges {1007616=573440,1581056=516096,2097152=286720} 2024-09-13T14:25:31.497 INFO:tasks.rados.rados.0.smithi026.stdout:1998: writing smithi02683976-42 from 1007616 to 1581056 tid 1 2024-09-13T14:25:31.500 INFO:tasks.rados.rados.0.smithi026.stdout:1998: writing smithi02683976-42 from 1581056 to 2097152 tid 2 2024-09-13T14:25:31.503 INFO:tasks.rados.rados.0.smithi026.stdout:1998: writing smithi02683976-42 from 2097152 to 2383872 tid 3 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1999: delete oid 40 current snap is 184 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1997: finishing copy_from to smithi02683976-14 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1997: got expected ENOENT (src dne) 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1997: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1996: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1995: finishing write tid 2 to smithi02683976-4 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1995: finishing write tid 3 to smithi02683976-4 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1996: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1995: finishing write tid 4 to smithi02683976-4 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1996: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 953 (ObjNum 605 snap 172 seq_num 605) dirty exists 2024-09-13T14:25:31.504 INFO:tasks.rados.rados.0.smithi026.stdout:1995: finishing write tid 5 to smithi02683976-4 2024-09-13T14:25:31.505 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 613 (ObjNum 654 snap 184 seq_num 654) dirty exists 2024-09-13T14:25:31.505 INFO:tasks.rados.rados.0.smithi026.stdout:1995: left oid 4 (ObjNum 654 snap 184 seq_num 654) 2024-09-13T14:25:31.505 INFO:tasks.rados.rados.0.smithi026.stdout:1998: finishing write tid 1 to smithi02683976-42 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:1995: done (4 left) 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:1996: done (3 left) 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:1997: done (2 left) 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:1999: done (1 left) 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:2000: read oid 26 snap -1 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:2000: expect (ObjNum 605 snap 172 seq_num 605) 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:2001: snap_create 2024-09-13T14:25:31.506 INFO:tasks.rados.rados.0.smithi026.stdout:1998: finishing write tid 2 to smithi02683976-42 2024-09-13T14:25:31.507 INFO:tasks.rados.rados.0.smithi026.stdout:1998: finishing write tid 3 to smithi02683976-42 2024-09-13T14:25:31.507 INFO:tasks.rados.rados.0.smithi026.stdout:1998: finishing write tid 4 to smithi02683976-42 2024-09-13T14:25:31.507 INFO:tasks.rados.rados.0.smithi026.stdout:1998: finishing write tid 5 to smithi02683976-42 2024-09-13T14:25:31.507 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 931 (ObjNum 655 snap 184 seq_num 655) dirty exists 2024-09-13T14:25:31.507 INFO:tasks.rados.rados.0.smithi026.stdout:1998: left oid 42 (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:31.509 INFO:tasks.rados.rados.0.smithi026.stdout:2000: expect (ObjNum 605 snap 172 seq_num 605) 2024-09-13T14:25:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:31] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.51.0" 2024-09-13T14:25:32.091 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 podman[120243]: 2024-09-13 14:25:31.76256378 +0000 UTC m=+0.680632472 container remove 03d404193292a5ba58bda39e32463895177e71e5f3b50d3c62558675b259b7dc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:25:32.091 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 podman[120306]: 2024-09-13 14:25:31.96327992 +0000 UTC m=+0.107293392 container create 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T14:25:32.091 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:31 smithi026 podman[120306]: 2024-09-13 14:25:31.872187898 +0000 UTC m=+0.016201383 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:1998: done (2 left) 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:2000: done (1 left) 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:2001: done (0 left) 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:2002: rollback oid 39 current snap is 185 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 178 2024-09-13T14:25:32.184 INFO:tasks.rados.rados.0.smithi026.stdout:2003: snap_create 2024-09-13T14:25:32.190 INFO:tasks.rados.rados.0.smithi026.stdout:2002: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:25:32.190 INFO:tasks.rados.rados.0.smithi026.stdout:2002: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:25:32.190 INFO:tasks.rados.rados.0.smithi026.stdout:2002: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:25:32.190 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 661 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:25:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:32 smithi145 ceph-mon[89145]: pgmap v250: 121 pgs: 1 active+undersized, 17 stale+active+clean, 2 active+undersized+degraded, 101 active+clean; 220 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 24 op/s; 11/441 objects degraded (2.494%) 2024-09-13T14:25:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:32 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 11/441 objects degraded (2.494%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:32 smithi145 ceph-mon[89145]: osdmap e435: 8 total, 7 up, 8 in 2024-09-13T14:25:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[99461]: pgmap v250: 121 pgs: 1 active+undersized, 17 stale+active+clean, 2 active+undersized+degraded, 101 active+clean; 220 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 24 op/s; 11/441 objects degraded (2.494%) 2024-09-13T14:25:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 11/441 objects degraded (2.494%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[99461]: osdmap e435: 8 total, 7 up, 8 in 2024-09-13T14:25:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[97022]: pgmap v250: 121 pgs: 1 active+undersized, 17 stale+active+clean, 2 active+undersized+degraded, 101 active+clean; 220 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.1 MiB/s wr, 24 op/s; 11/441 objects degraded (2.494%) 2024-09-13T14:25:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 11/441 objects degraded (2.494%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-mon[97022]: osdmap e435: 8 total, 7 up, 8 in 2024-09-13T14:25:32.666 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:32 smithi026 podman[120306]: 2024-09-13 14:25:32.174061138 +0000 UTC m=+0.318074611 container init 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T14:25:32.666 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:32 smithi026 podman[120306]: 2024-09-13 14:25:32.179647301 +0000 UTC m=+0.323660781 container start 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T14:25:32.666 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:32 smithi026 ceph-osd[120344]: -- 172.21.15.26:0/4139878484 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56216619fa40 con 0x56216617dc00 2024-09-13T14:25:32.666 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:32 smithi026 bash[120306]: 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 2024-09-13T14:25:32.667 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:32 smithi026 systemd[1]: Started Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:25:33.190 INFO:tasks.rados.rados.0.smithi026.stdout:2002: done (1 left) 2024-09-13T14:25:33.190 INFO:tasks.rados.rados.0.smithi026.stdout:2003: done (0 left) 2024-09-13T14:25:33.191 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 186 2024-09-13T14:25:33.191 INFO:tasks.rados.rados.0.smithi026.stdout:2004: seq_num 657 ranges {3981312=606208,4587520=688128,5275648=114688} 2024-09-13T14:25:33.215 INFO:tasks.rados.rados.0.smithi026.stdout:2004: writing smithi02683976-28 from 3981312 to 4587520 tid 1 2024-09-13T14:25:33.219 INFO:tasks.rados.rados.0.smithi026.stdout:2004: writing smithi02683976-28 from 4587520 to 5275648 tid 2 2024-09-13T14:25:33.220 INFO:tasks.rados.rados.0.smithi026.stdout:2004: writing smithi02683976-28 from 5275648 to 5390336 tid 3 2024-09-13T14:25:33.220 INFO:tasks.rados.rados.0.smithi026.stdout:2005: snap_create 2024-09-13T14:25:33.221 INFO:tasks.rados.rados.0.smithi026.stdout:2004: finishing write tid 1 to smithi02683976-28 2024-09-13T14:25:33.223 INFO:tasks.rados.rados.0.smithi026.stdout:2004: finishing write tid 2 to smithi02683976-28 2024-09-13T14:25:33.223 INFO:tasks.rados.rados.0.smithi026.stdout:2004: finishing write tid 3 to smithi02683976-28 2024-09-13T14:25:33.223 INFO:tasks.rados.rados.0.smithi026.stdout:2004: finishing write tid 4 to smithi02683976-28 2024-09-13T14:25:33.225 INFO:tasks.rados.rados.0.smithi026.stdout:2004: finishing write tid 5 to smithi02683976-28 2024-09-13T14:25:33.225 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 836 (ObjNum 657 snap 186 seq_num 657) dirty exists 2024-09-13T14:25:33.225 INFO:tasks.rados.rados.0.smithi026.stdout:2004: left oid 28 (ObjNum 657 snap 186 seq_num 657) 2024-09-13T14:25:33.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[99461]: osdmap e436: 8 total, 7 up, 8 in 2024-09-13T14:25:33.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[97022]: osdmap e436: 8 total, 7 up, 8 in 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:33 smithi145 ceph-mon[89145]: osdmap e436: 8 total, 7 up, 8 in 2024-09-13T14:25:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:33 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:33.925 INFO:tasks.rados.rados.0.smithi026.stdout:2004: done (1 left) 2024-09-13T14:25:33.926 INFO:tasks.rados.rados.0.smithi026.stdout:2005: done (0 left) 2024-09-13T14:25:33.926 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 187 2024-09-13T14:25:33.926 INFO:tasks.rados.rados.0.smithi026.stdout:2006: seq_num 658 ranges {737280=622592,1359872=565248,1925120=237568} 2024-09-13T14:25:33.932 INFO:tasks.rados.rados.0.smithi026.stdout:2006: writing smithi02683976-22 from 737280 to 1359872 tid 1 2024-09-13T14:25:33.936 INFO:tasks.rados.rados.0.smithi026.stdout:2006: writing smithi02683976-22 from 1359872 to 1925120 tid 2 2024-09-13T14:25:33.937 INFO:tasks.rados.rados.0.smithi026.stdout:2006: writing smithi02683976-22 from 1925120 to 2162688 tid 3 2024-09-13T14:25:33.937 INFO:tasks.rados.rados.0.smithi026.stdout:2007: copy_from oid 50 from oid 15 current snap is 187 2024-09-13T14:25:33.937 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 187 2024-09-13T14:25:33.938 INFO:tasks.rados.rados.0.smithi026.stdout:2008: seq_num 659 ranges {1056768=647168,1703936=450560,2154496=352256} 2024-09-13T14:25:33.947 INFO:tasks.rados.rados.0.smithi026.stdout:2008: writing smithi02683976-8 from 1056768 to 1703936 tid 1 2024-09-13T14:25:33.951 INFO:tasks.rados.rados.0.smithi026.stdout:2008: writing smithi02683976-8 from 1703936 to 2154496 tid 2 2024-09-13T14:25:33.953 INFO:tasks.rados.rados.0.smithi026.stdout:2008: writing smithi02683976-8 from 2154496 to 2506752 tid 3 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2009: copy_from oid 31 from oid 27 current snap is 187 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: finishing write tid 1 to smithi02683976-22 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: finishing write tid 2 to smithi02683976-22 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: finishing write tid 3 to smithi02683976-22 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: finishing write tid 4 to smithi02683976-22 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: finishing write tid 5 to smithi02683976-22 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 957 (ObjNum 658 snap 187 seq_num 658) dirty exists 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2006: left oid 22 (ObjNum 658 snap 187 seq_num 658) 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2007: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2007: finishing copy_from to smithi02683976-50 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 609 (ObjNum 651 snap 183 seq_num 651) dirty exists 2024-09-13T14:25:33.954 INFO:tasks.rados.rados.0.smithi026.stdout:2008: finishing write tid 1 to smithi02683976-8 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2008: finishing write tid 2 to smithi02683976-8 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2006: done (3 left) 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2007: done (2 left) 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2010: snap_create 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2008: finishing write tid 3 to smithi02683976-8 2024-09-13T14:25:33.955 INFO:tasks.rados.rados.0.smithi026.stdout:2008: finishing write tid 4 to smithi02683976-8 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:2009: finishing copy_from to smithi02683976-31 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:2009: got expected ENOENT (src dne) 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:2009: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:2008: finishing write tid 5 to smithi02683976-8 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 592 (ObjNum 659 snap 187 seq_num 659) dirty exists 2024-09-13T14:25:33.957 INFO:tasks.rados.rados.0.smithi026.stdout:2008: left oid 8 (ObjNum 659 snap 187 seq_num 659) 2024-09-13T14:25:34.149 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:25:33.841+0000 7ff276362640 -1 Falling back to public interface 2024-09-13T14:25:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[97022]: pgmap v253: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 49 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[97022]: osdmap e437: 8 total, 7 up, 8 in 2024-09-13T14:25:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[97022]: osdmap e438: 8 total, 7 up, 8 in 2024-09-13T14:25:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[99461]: pgmap v253: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 49 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[99461]: osdmap e437: 8 total, 7 up, 8 in 2024-09-13T14:25:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:34 smithi026 ceph-mon[99461]: osdmap e438: 8 total, 7 up, 8 in 2024-09-13T14:25:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:34 smithi145 ceph-mon[89145]: pgmap v253: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.5 MiB/s wr, 49 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:34 smithi145 ceph-mon[89145]: osdmap e437: 8 total, 7 up, 8 in 2024-09-13T14:25:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:34 smithi145 ceph-mon[89145]: osdmap e438: 8 total, 7 up, 8 in 2024-09-13T14:25:34.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:34.926 INFO:tasks.rados.rados.0.smithi026.stdout:2008: done (2 left) 2024-09-13T14:25:34.926 INFO:tasks.rados.rados.0.smithi026.stdout:2009: done (1 left) 2024-09-13T14:25:34.926 INFO:tasks.rados.rados.0.smithi026.stdout:2010: done (0 left) 2024-09-13T14:25:34.926 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 188 2024-09-13T14:25:34.926 INFO:tasks.rados.rados.0.smithi026.stdout:2011: seq_num 660 ranges {573440=663552,1236992=737280,1974272=65536} 2024-09-13T14:25:34.933 INFO:tasks.rados.rados.0.smithi026.stdout:2011: writing smithi02683976-43 from 573440 to 1236992 tid 1 2024-09-13T14:25:34.937 INFO:tasks.rados.rados.0.smithi026.stdout:2011: writing smithi02683976-43 from 1236992 to 1974272 tid 2 2024-09-13T14:25:34.937 INFO:tasks.rados.rados.0.smithi026.stdout:2011: writing smithi02683976-43 from 1974272 to 2039808 tid 3 2024-09-13T14:25:34.938 INFO:tasks.rados.rados.0.smithi026.stdout:2012: delete oid 18 current snap is 188 2024-09-13T14:25:34.942 INFO:tasks.rados.rados.0.smithi026.stdout:2011: finishing write tid 1 to smithi02683976-43 2024-09-13T14:25:34.942 INFO:tasks.rados.rados.0.smithi026.stdout:2011: finishing write tid 2 to smithi02683976-43 2024-09-13T14:25:34.943 INFO:tasks.rados.rados.0.smithi026.stdout:2011: finishing write tid 3 to smithi02683976-43 2024-09-13T14:25:34.944 INFO:tasks.rados.rados.0.smithi026.stdout:2011: finishing write tid 4 to smithi02683976-43 2024-09-13T14:25:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:2012: done (1 left) 2024-09-13T14:25:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:2013: read oid 13 snap -1 2024-09-13T14:25:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:2013: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:25:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 188 2024-09-13T14:25:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:2014: seq_num 662 ranges {3170304=696320,3866624=557056,4423680=245760} 2024-09-13T14:25:34.966 INFO:tasks.rados.rados.0.smithi026.stdout:2014: writing smithi02683976-39 from 3170304 to 3866624 tid 1 2024-09-13T14:25:34.969 INFO:tasks.rados.rados.0.smithi026.stdout:2014: writing smithi02683976-39 from 3866624 to 4423680 tid 2 2024-09-13T14:25:34.971 INFO:tasks.rados.rados.0.smithi026.stdout:2014: writing smithi02683976-39 from 4423680 to 4669440 tid 3 2024-09-13T14:25:34.971 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 188 2024-09-13T14:25:34.971 INFO:tasks.rados.rados.0.smithi026.stdout:2015: seq_num 663 ranges {0=712704,712704=442368,1155072=360448} 2024-09-13T14:25:34.975 INFO:tasks.rados.rados.0.smithi026.stdout:2015: writing smithi02683976-30 from 0 to 712704 tid 1 2024-09-13T14:25:34.977 INFO:tasks.rados.rados.0.smithi026.stdout:2015: writing smithi02683976-30 from 712704 to 1155072 tid 2 2024-09-13T14:25:34.980 INFO:tasks.rados.rados.0.smithi026.stdout:2015: writing smithi02683976-30 from 1155072 to 1515520 tid 3 2024-09-13T14:25:34.981 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 188 2024-09-13T14:25:34.981 INFO:tasks.rados.rados.0.smithi026.stdout:2016: seq_num 664 ranges {2506752=729088,3235840=729088,3964928=73728} 2024-09-13T14:25:34.998 INFO:tasks.rados.rados.0.smithi026.stdout:2016: writing smithi02683976-8 from 2506752 to 3235840 tid 1 2024-09-13T14:25:35.002 INFO:tasks.rados.rados.0.smithi026.stdout:2016: writing smithi02683976-8 from 3235840 to 3964928 tid 2 2024-09-13T14:25:35.003 INFO:tasks.rados.rados.0.smithi026.stdout:2016: writing smithi02683976-8 from 3964928 to 4038656 tid 3 2024-09-13T14:25:35.003 INFO:tasks.rados.rados.0.smithi026.stdout:2017: read oid 1 snap -1 2024-09-13T14:25:35.003 INFO:tasks.rados.rados.0.smithi026.stdout:2017: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:25:35.003 INFO:tasks.rados.rados.0.smithi026.stdout:2018: snap_remove snap 181 2024-09-13T14:25:35.932 INFO:tasks.rados.rados.0.smithi026.stdout:2018: done (6 left) 2024-09-13T14:25:35.932 INFO:tasks.rados.rados.0.smithi026.stdout:2019: rollback oid 4 current snap is 188 2024-09-13T14:25:35.932 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 166 2024-09-13T14:25:35.932 INFO:tasks.rados.rados.0.smithi026.stdout:2013: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:25:35.936 INFO:tasks.rados.rados.0.smithi026.stdout:2013: done (6 left) 2024-09-13T14:25:35.936 INFO:tasks.rados.rados.0.smithi026.stdout:2020: read oid 9 snap -1 2024-09-13T14:25:35.936 INFO:tasks.rados.rados.0.smithi026.stdout:2020: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:25:35.936 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 188 2024-09-13T14:25:35.937 INFO:tasks.rados.rados.0.smithi026.stdout:2021: seq_num 665 ranges {6905856=745472,7651328=606208,8257536=196608} 2024-09-13T14:25:35.980 INFO:tasks.rados.rados.0.smithi026.stdout:2021: writing smithi02683976-5 from 6905856 to 7651328 tid 1 2024-09-13T14:25:35.985 INFO:tasks.rados.rados.0.smithi026.stdout:2021: writing smithi02683976-5 from 7651328 to 8257536 tid 2 2024-09-13T14:25:35.987 INFO:tasks.rados.rados.0.smithi026.stdout:2021: writing smithi02683976-5 from 8257536 to 8454144 tid 3 2024-09-13T14:25:35.987 INFO:tasks.rados.rados.0.smithi026.stdout:2022: delete oid 36 current snap is 188 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2011: finishing write tid 5 to smithi02683976-43 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 974 (ObjNum 660 snap 188 seq_num 660) dirty exists 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2011: left oid 43 (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: finishing write tid 1 to smithi02683976-39 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: finishing write tid 2 to smithi02683976-39 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: finishing write tid 3 to smithi02683976-39 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: finishing write tid 4 to smithi02683976-39 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: finishing write tid 5 to smithi02683976-39 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 665 (ObjNum 662 snap 188 seq_num 662) dirty exists 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2014: left oid 39 (ObjNum 662 snap 188 seq_num 662) 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2015: finishing write tid 1 to smithi02683976-30 2024-09-13T14:25:35.988 INFO:tasks.rados.rados.0.smithi026.stdout:2015: finishing write tid 2 to smithi02683976-30 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2015: finishing write tid 3 to smithi02683976-30 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2015: finishing write tid 4 to smithi02683976-30 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2015: finishing write tid 5 to smithi02683976-30 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 617 (ObjNum 663 snap 188 seq_num 663) dirty exists 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2015: left oid 30 (ObjNum 663 snap 188 seq_num 663) 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: finishing write tid 1 to smithi02683976-8 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: finishing write tid 2 to smithi02683976-8 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: finishing write tid 3 to smithi02683976-8 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: finishing write tid 4 to smithi02683976-8 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: finishing write tid 5 to smithi02683976-8 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 596 (ObjNum 664 snap 188 seq_num 664) dirty exists 2024-09-13T14:25:35.989 INFO:tasks.rados.rados.0.smithi026.stdout:2016: left oid 8 (ObjNum 664 snap 188 seq_num 664) 2024-09-13T14:25:35.990 INFO:tasks.rados.rados.0.smithi026.stdout:2017: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:25:36.021 INFO:tasks.rados.rados.0.smithi026.stdout:2019: finishing rollback tid 0 to smithi02683976-4 2024-09-13T14:25:36.022 INFO:tasks.rados.rados.0.smithi026.stdout:2019: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:25:36.022 INFO:tasks.rados.rados.0.smithi026.stdout:2019: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:25:36.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 620 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:25:36.022 INFO:tasks.rados.rados.0.smithi026.stdout:2020: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:25:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:35 smithi145 ceph-mon[89145]: pgmap v256: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 30 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:35 smithi145 ceph-mon[89145]: osdmap e439: 8 total, 7 up, 8 in 2024-09-13T14:25:36.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:36.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: finishing write tid 1 to smithi02683976-5 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: finishing write tid 2 to smithi02683976-5 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: finishing write tid 3 to smithi02683976-5 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: finishing write tid 4 to smithi02683976-5 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: finishing write tid 5 to smithi02683976-5 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 993 (ObjNum 665 snap 188 seq_num 665) dirty exists 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2021: left oid 5 (ObjNum 665 snap 188 seq_num 665) 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2011: done (8 left) 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2014: done (7 left) 2024-09-13T14:25:36.210 INFO:tasks.rados.rados.0.smithi026.stdout:2015: done (6 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2016: done (5 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2017: done (4 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2019: done (3 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2020: done (2 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2021: done (1 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2022: done (0 left) 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 188 2024-09-13T14:25:36.211 INFO:tasks.rados.rados.0.smithi026.stdout:2023: seq_num 667 ranges {5160960=778240,5939200=778240,6717440=24576} 2024-09-13T14:25:36.242 INFO:tasks.rados.rados.0.smithi026.stdout:2023: writing smithi02683976-46 from 5160960 to 5939200 tid 1 2024-09-13T14:25:36.246 INFO:tasks.rados.rados.0.smithi026.stdout:2023: writing smithi02683976-46 from 5939200 to 6717440 tid 2 2024-09-13T14:25:36.247 INFO:tasks.rados.rados.0.smithi026.stdout:2023: writing smithi02683976-46 from 6717440 to 6742016 tid 3 2024-09-13T14:25:36.247 INFO:tasks.rados.rados.0.smithi026.stdout:2024: read oid 31 snap -1 2024-09-13T14:25:36.247 INFO:tasks.rados.rados.0.smithi026.stdout:2024: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:25:36.247 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 188 2024-09-13T14:25:36.248 INFO:tasks.rados.rados.0.smithi026.stdout:2023: finishing write tid 1 to smithi02683976-46 2024-09-13T14:25:36.248 INFO:tasks.rados.rados.0.smithi026.stdout:2025: seq_num 668 ranges {958464=794624,1753088=663552,2416640=139264} 2024-09-13T14:25:36.256 INFO:tasks.rados.rados.0.smithi026.stdout:2025: writing smithi02683976-47 from 958464 to 1753088 tid 1 2024-09-13T14:25:36.260 INFO:tasks.rados.rados.0.smithi026.stdout:2025: writing smithi02683976-47 from 1753088 to 2416640 tid 2 2024-09-13T14:25:36.261 INFO:tasks.rados.rados.0.smithi026.stdout:2025: writing smithi02683976-47 from 2416640 to 2555904 tid 3 2024-09-13T14:25:36.262 INFO:tasks.rados.rados.0.smithi026.stdout:2026: read oid 22 snap -1 2024-09-13T14:25:36.262 INFO:tasks.rados.rados.0.smithi026.stdout:2026: expect (ObjNum 658 snap 187 seq_num 658) 2024-09-13T14:25:36.262 INFO:tasks.rados.rados.0.smithi026.stdout:2027: snap_remove snap 166 2024-09-13T14:25:36.366 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:25:36.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:35 smithi026 ceph-mon[97022]: pgmap v256: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 30 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:36.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:35 smithi026 ceph-mon[97022]: osdmap e439: 8 total, 7 up, 8 in 2024-09-13T14:25:36.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:35 smithi026 ceph-mon[99461]: pgmap v256: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 30 op/s; 28/477 objects degraded (5.870%) 2024-09-13T14:25:36.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:35 smithi026 ceph-mon[99461]: osdmap e439: 8 total, 7 up, 8 in 2024-09-13T14:25:36.367 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:25:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:25:35.994+0000 7f450bb4b640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 116 after trimming was started 2024-09-13T14:25:36.943 INFO:tasks.rados.rados.0.smithi026.stdout:2027: done (4 left) 2024-09-13T14:25:36.943 INFO:tasks.rados.rados.0.smithi026.stdout:2028: copy_from oid 24 from oid 25 current snap is 188 2024-09-13T14:25:36.943 INFO:tasks.rados.rados.0.smithi026.stdout:2029: copy_from oid 28 from oid 42 current snap is 188 2024-09-13T14:25:36.943 INFO:tasks.rados.rados.0.smithi026.stdout:2030: snap_remove snap 182 2024-09-13T14:25:36.983 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:25:36.983 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 1s ago 9m 18.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (3m) 2m ago 8m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (10m) 1s ago 10m 15.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (10m) 2m ago 10m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (3m) 2m ago 13m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (3m) 1s ago 15m 523M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 1s ago 15m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (2m) 2m ago 14m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:25:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 1s ago 14m 46.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 1s ago 8m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (3m) 2m ago 8m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (106s) 1s ago 13m 152M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (71s) 1s ago 13m 144M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (38s) 1s ago 12m 108M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (5s) 1s ago 12m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:25:36.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (12m) 2m ago 12m 171M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 74b38ce01808 2024-09-13T14:25:36.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (11m) 2m ago 11m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:25:36.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (11m) 2m ago 11m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:25:36.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (10m) 2m ago 10m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:25:36.986 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (3m) 2m ago 9m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:25:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: osdmap e440: 8 total, 7 up, 8 in 2024-09-13T14:25:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: osdmap e440: 8 total, 7 up, 8 in 2024-09-13T14:25:37.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: osdmap e440: 8 total, 7 up, 8 in 2024-09-13T14:25:37.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:37.346 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:25:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:25:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:25:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-13T14:25:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 8 2024-09-13T14:25:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:25:37.348 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:25:37.659 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:25:37.659 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:25:37.659 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:25:37.659 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:25:37.659 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "9/20 daemons upgraded", 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:25:37.660 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:25:37.808 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:25:37.497+0000 7ff276362640 -1 osd.3 430 log_to_monitors true 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2030: done (6 left) 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2031: setattr oid 48 current snap is 188 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2023: finishing write tid 2 to smithi02683976-46 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2023: finishing write tid 3 to smithi02683976-46 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2023: finishing write tid 4 to smithi02683976-46 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:2023: finishing write tid 5 to smithi02683976-46 2024-09-13T14:25:37.943 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 470 (ObjNum 667 snap 188 seq_num 667) dirty exists 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2023: left oid 46 (ObjNum 667 snap 188 seq_num 667) 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: finishing write tid 1 to smithi02683976-47 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: finishing write tid 2 to smithi02683976-47 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: finishing write tid 3 to smithi02683976-47 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: finishing write tid 4 to smithi02683976-47 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: finishing write tid 5 to smithi02683976-47 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 740 (ObjNum 668 snap 188 seq_num 668) dirty exists 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2025: left oid 47 (ObjNum 668 snap 188 seq_num 668) 2024-09-13T14:25:37.944 INFO:tasks.rados.rados.0.smithi026.stdout:2024: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:25:38.108 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:25:37.968+0000 7ff26e10c640 -1 osd.3 430 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: from='client.44249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: pgmap v259: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.7 MiB/s wr, 18 op/s; 30/507 objects degraded (5.917%) 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: osdmap e441: 8 total, 7 up, 8 in 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4098173723' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:38.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 30/507 objects degraded (5.917%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: from='client.44249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: pgmap v259: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.7 MiB/s wr, 18 op/s; 30/507 objects degraded (5.917%) 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: osdmap e441: 8 total, 7 up, 8 in 2024-09-13T14:25:38.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4098173723' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:37 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 30/507 objects degraded (5.917%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: from='client.44249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: pgmap v259: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 238 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.7 MiB/s wr, 18 op/s; 30/507 objects degraded (5.917%) 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: osdmap e441: 8 total, 7 up, 8 in 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4098173723' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:25:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:37 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 30/507 objects degraded (5.917%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:25:38.283 INFO:tasks.rados.rados.0.smithi026.stdout:2026: expect (ObjNum 658 snap 187 seq_num 658) 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2029: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2029: finishing copy_from to smithi02683976-28 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 931 (ObjNum 655 snap 184 seq_num 655) dirty exists 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2028: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2028: finishing copy_from to smithi02683976-24 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 658 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2023: done (6 left) 2024-09-13T14:25:38.353 INFO:tasks.rados.rados.0.smithi026.stdout:2024: done (5 left) 2024-09-13T14:25:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2025: done (4 left) 2024-09-13T14:25:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2026: done (3 left) 2024-09-13T14:25:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2028: done (2 left) 2024-09-13T14:25:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2029: done (1 left) 2024-09-13T14:25:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2032: setattr oid 24 current snap is 188 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2033: read oid 23 snap 173 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2033: expect (ObjNum 602 snap 172 seq_num 602) 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2034: rmattr oid 46 current snap is 188 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2034: done (3 left) 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2035: copy_from oid 46 from oid 38 current snap is 188 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2036: rollback oid 3 current snap is 188 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 185 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 188 2024-09-13T14:25:38.355 INFO:tasks.rados.rados.0.smithi026.stdout:2037: seq_num 669 ranges {1515520=409600,1925120=548864,2473984=655360} 2024-09-13T14:25:38.364 INFO:tasks.rados.rados.0.smithi026.stdout:2037: writing smithi02683976-30 from 1515520 to 1925120 tid 1 2024-09-13T14:25:38.367 INFO:tasks.rados.rados.0.smithi026.stdout:2037: writing smithi02683976-30 from 1925120 to 2473984 tid 2 2024-09-13T14:25:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:2037: writing smithi02683976-30 from 2473984 to 3129344 tid 3 2024-09-13T14:25:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:2038: setattr oid 9 current snap is 188 2024-09-13T14:25:38.371 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 741 (ObjNum 609 snap 173 seq_num 609) dirty exists 2024-09-13T14:25:38.372 INFO:tasks.rados.rados.0.smithi026.stdout:2036: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:25:38.372 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 964 (ObjNum 193 snap 0 seq_num 519154176) dirty dne 2024-09-13T14:25:38.372 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 666 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:25:38.372 INFO:tasks.rados.rados.0.smithi026.stdout:2037: finishing write tid 1 to smithi02683976-30 2024-09-13T14:25:38.372 INFO:tasks.rados.rados.0.smithi026.stdout:2037: finishing write tid 2 to smithi02683976-30 2024-09-13T14:25:38.373 INFO:tasks.rados.rados.0.smithi026.stdout:2031: done (6 left) 2024-09-13T14:25:38.373 INFO:tasks.rados.rados.0.smithi026.stdout:2032: done (5 left) 2024-09-13T14:25:38.373 INFO:tasks.rados.rados.0.smithi026.stdout:2036: done (4 left) 2024-09-13T14:25:38.373 INFO:tasks.rados.rados.0.smithi026.stdout:2039: rollback oid 33 current snap is 188 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 183 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:2040: read oid 24 snap -1 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:2040: expect (ObjNum 647 snap 182 seq_num 647) 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:2041: read oid 28 snap -1 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:2041: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:38.374 INFO:tasks.rados.rados.0.smithi026.stdout:2042: setattr oid 31 current snap is 188 2024-09-13T14:25:38.375 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 188 2024-09-13T14:25:38.376 INFO:tasks.rados.rados.0.smithi026.stdout:2043: seq_num 670 ranges {5808128=425984,6234112=483328,6717440=507904,7225344=212992} 2024-09-13T14:25:38.411 INFO:tasks.rados.rados.0.smithi026.stdout:2043: writing smithi02683976-16 from 5808128 to 6234112 tid 1 2024-09-13T14:25:38.413 INFO:tasks.rados.rados.0.smithi026.stdout:2043: writing smithi02683976-16 from 6234112 to 6717440 tid 2 2024-09-13T14:25:38.417 INFO:tasks.rados.rados.0.smithi026.stdout:2043: writing smithi02683976-16 from 6717440 to 7225344 tid 3 2024-09-13T14:25:38.418 INFO:tasks.rados.rados.0.smithi026.stdout:2043: writing smithi02683976-16 from 7225344 to 7438336 tid 4 2024-09-13T14:25:38.418 INFO:tasks.rados.rados.0.smithi026.stdout:2044: snap_remove snap 186 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2044: done (9 left) 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2045: snap_create 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2037: finishing write tid 3 to smithi02683976-30 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2037: finishing write tid 4 to smithi02683976-30 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2037: finishing write tid 5 to smithi02683976-30 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 624 (ObjNum 669 snap 188 seq_num 669) dirty exists 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:2037: left oid 30 (ObjNum 669 snap 188 seq_num 669) 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1005 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:25:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1007 (ObjNum 614 snap 176 seq_num 614) dirty exists 2024-09-13T14:25:38.969 INFO:tasks.rados.rados.0.smithi026.stdout:2033: expect (ObjNum 602 snap 172 seq_num 602) 2024-09-13T14:25:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:38 smithi145 ceph-mon[89145]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:38 smithi145 ceph-mon[89145]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:25:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:38 smithi145 ceph-mon[89145]: osdmap e442: 8 total, 7 up, 8 in 2024-09-13T14:25:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:38 smithi145 ceph-mon[89145]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:38 smithi145 ceph-mon[89145]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[99461]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[99461]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[99461]: osdmap e442: 8 total, 7 up, 8 in 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[99461]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[99461]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[97022]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[97022]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:25:39.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[97022]: osdmap e442: 8 total, 7 up, 8 in 2024-09-13T14:25:39.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[97022]: from='osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:38 smithi026 ceph-mon[97022]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]} : dispatch 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:2039: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:2039: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:2039: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1012 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:2035: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:25:39.533 INFO:tasks.rados.rados.0.smithi026.stdout:2041: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:39.612 INFO:tasks.rados.rados.0.smithi026.stdout:2040: expect (ObjNum 647 snap 182 seq_num 647) 2024-09-13T14:25:39.911 INFO:tasks.rados.rados.0.smithi026.stdout:2035: finishing copy_from to smithi02683976-46 2024-09-13T14:25:39.911 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 920 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:25:39.911 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 1 to smithi02683976-16 2024-09-13T14:25:39.911 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 2 to smithi02683976-16 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 3 to smithi02683976-16 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 4 to smithi02683976-16 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 5 to smithi02683976-16 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:2043: finishing write tid 6 to smithi02683976-16 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 979 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:25:39.912 INFO:tasks.rados.rados.0.smithi026.stdout:2043: left oid 16 (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2033: done (9 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2035: done (8 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2037: done (7 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2038: done (6 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2039: done (5 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2040: done (4 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2041: done (3 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2042: done (2 left) 2024-09-13T14:25:39.963 INFO:tasks.rados.rados.0.smithi026.stdout:2043: done (1 left) 2024-09-13T14:25:39.964 INFO:tasks.rados.rados.0.smithi026.stdout:2045: done (0 left) 2024-09-13T14:25:39.964 INFO:tasks.rados.rados.0.smithi026.stdout:2046: delete oid 38 current snap is 189 2024-09-13T14:25:39.982 INFO:tasks.rados.rados.0.smithi026.stdout:2046: done (0 left) 2024-09-13T14:25:39.983 INFO:tasks.rados.rados.0.smithi026.stdout:2047: snap_remove snap 178 2024-09-13T14:25:40.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:25:40.020+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: pgmap v262: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 22 op/s; 32/507 objects degraded (6.312%) 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605] boot 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: pgmap v262: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 22 op/s; 32/507 objects degraded (6.312%) 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605] boot 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: pgmap v262: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 22 op/s; 32/507 objects degraded (6.312%) 2024-09-13T14:25:40.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: osd.3 [v2:172.21.15.26:6826/3139338605,v1:172.21.15.26:6827/3139338605] boot 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:40.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:40.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:40.974 INFO:tasks.rados.rados.0.smithi026.stdout:2047: done (0 left) 2024-09-13T14:25:40.974 INFO:tasks.rados.rados.0.smithi026.stdout:2048: rmattr oid 12 current snap is 189 2024-09-13T14:25:40.974 INFO:tasks.rados.rados.0.smithi026.stdout:2048: done (0 left) 2024-09-13T14:25:40.974 INFO:tasks.rados.rados.0.smithi026.stdout:2049: snap_remove snap 187 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:41.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:41.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:40 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:25:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:41.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:41.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:40 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T14:25:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:41] "GET /metrics HTTP/1.1" 200 36095 "" "Prometheus/2.51.0" 2024-09-13T14:25:41.972 INFO:tasks.rados.rados.0.smithi026.stdout:2049: done (0 left) 2024-09-13T14:25:41.972 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 189 2024-09-13T14:25:41.972 INFO:tasks.rados.rados.0.smithi026.stdout:2050: seq_num 672 ranges {8388608=458752,8847360=647168,9494528=466944,9961472=90112} 2024-09-13T14:25:42.022 INFO:tasks.rados.rados.0.smithi026.stdout:2050: writing smithi02683976-34 from 8388608 to 8847360 tid 1 2024-09-13T14:25:42.024 INFO:tasks.rados.rados.0.smithi026.stdout:2050: writing smithi02683976-34 from 8847360 to 9494528 tid 2 2024-09-13T14:25:42.027 INFO:tasks.rados.rados.0.smithi026.stdout:2050: writing smithi02683976-34 from 9494528 to 9961472 tid 3 2024-09-13T14:25:42.028 INFO:tasks.rados.rados.0.smithi026.stdout:2050: writing smithi02683976-34 from 9961472 to 10051584 tid 4 2024-09-13T14:25:42.028 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 189 2024-09-13T14:25:42.028 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 1 to smithi02683976-34 2024-09-13T14:25:42.028 INFO:tasks.rados.rados.0.smithi026.stdout:2051: seq_num 673 ranges {0=475136,475136=532480,1007616=671744} 2024-09-13T14:25:42.030 INFO:tasks.rados.rados.0.smithi026.stdout:2051: writing smithi02683976-38 from 0 to 475136 tid 1 2024-09-13T14:25:42.034 INFO:tasks.rados.rados.0.smithi026.stdout:2051: writing smithi02683976-38 from 475136 to 1007616 tid 2 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2051: writing smithi02683976-38 from 1007616 to 1679360 tid 3 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 189 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 2 to smithi02683976-34 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 3 to smithi02683976-34 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 4 to smithi02683976-34 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 5 to smithi02683976-34 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:2050: finishing write tid 6 to smithi02683976-34 2024-09-13T14:25:42.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 818 (ObjNum 672 snap 189 seq_num 672) dirty exists 2024-09-13T14:25:42.040 INFO:tasks.rados.rados.0.smithi026.stdout:2050: left oid 34 (ObjNum 672 snap 189 seq_num 672) 2024-09-13T14:25:42.040 INFO:tasks.rados.rados.0.smithi026.stdout:2051: finishing write tid 1 to smithi02683976-38 2024-09-13T14:25:42.040 INFO:tasks.rados.rados.0.smithi026.stdout:2051: finishing write tid 2 to smithi02683976-38 2024-09-13T14:25:42.040 INFO:tasks.rados.rados.0.smithi026.stdout:2052: seq_num 674 ranges {1056768=491520,1548288=417792,1966080=483328,2449408=303104} 2024-09-13T14:25:42.046 INFO:tasks.rados.rados.0.smithi026.stdout:2052: writing smithi02683976-1 from 1056768 to 1548288 tid 1 2024-09-13T14:25:42.049 INFO:tasks.rados.rados.0.smithi026.stdout:2052: writing smithi02683976-1 from 1548288 to 1966080 tid 2 2024-09-13T14:25:42.053 INFO:tasks.rados.rados.0.smithi026.stdout:2052: writing smithi02683976-1 from 1966080 to 2449408 tid 3 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2052: writing smithi02683976-1 from 2449408 to 2752512 tid 4 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2050: done (2 left) 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2053: rollback oid 48 current snap is 189 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 183 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2051: finishing write tid 3 to smithi02683976-38 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2051: finishing write tid 4 to smithi02683976-38 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2051: finishing write tid 5 to smithi02683976-38 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 945 (ObjNum 673 snap 189 seq_num 673) dirty exists 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2051: left oid 38 (ObjNum 673 snap 189 seq_num 673) 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 1 to smithi02683976-1 2024-09-13T14:25:42.055 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 2 to smithi02683976-1 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2051: done (2 left) 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2054: rollback oid 9 current snap is 189 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 184 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2055: copy_from oid 7 from oid 34 current snap is 189 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2056: read oid 46 snap -1 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2056: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:42.056 INFO:tasks.rados.rados.0.smithi026.stdout:2057: delete oid 44 current snap is 189 2024-09-13T14:25:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 3 to smithi02683976-1 2024-09-13T14:25:42.058 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 4 to smithi02683976-1 2024-09-13T14:25:42.058 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 5 to smithi02683976-1 2024-09-13T14:25:42.058 INFO:tasks.rados.rados.0.smithi026.stdout:2053: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:25:42.059 INFO:tasks.rados.rados.0.smithi026.stdout:2053: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:25:42.059 INFO:tasks.rados.rados.0.smithi026.stdout:2054: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:25:42.060 INFO:tasks.rados.rados.0.smithi026.stdout:2052: finishing write tid 6 to smithi02683976-1 2024-09-13T14:25:42.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1034 (ObjNum 674 snap 189 seq_num 674) dirty exists 2024-09-13T14:25:42.060 INFO:tasks.rados.rados.0.smithi026.stdout:2052: left oid 1 (ObjNum 674 snap 189 seq_num 674) 2024-09-13T14:25:42.060 INFO:tasks.rados.rados.0.smithi026.stdout:2054: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:25:42.060 INFO:tasks.rados.rados.0.smithi026.stdout:2054: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:25:42.061 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1025 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:25:42.061 INFO:tasks.rados.rados.0.smithi026.stdout:2053: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:25:42.061 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 744 (ObjNum 609 snap 173 seq_num 609) dirty exists 2024-09-13T14:25:42.085 INFO:tasks.rados.rados.0.smithi026.stdout:2056: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:42.239 INFO:tasks.rados.rados.0.smithi026.stdout:2055: finishing copy_from racing read to smithi02683976-7 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2055: finishing copy_from to smithi02683976-7 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 818 (ObjNum 672 snap 189 seq_num 672) dirty exists 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2052: done (5 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2053: done (4 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2054: done (3 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2055: done (2 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2056: done (1 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2057: done (0 left) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2058: read oid 10 snap -1 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2058: expect (ObjNum 599 snap 171 seq_num 599) 2024-09-13T14:25:42.240 INFO:tasks.rados.rados.0.smithi026.stdout:2059: snap_remove snap 176 2024-09-13T14:25:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[99461]: pgmap v265: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.7 MiB/s wr, 15 op/s; 19/489 objects degraded (3.885%) 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[99461]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[97022]: pgmap v265: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.7 MiB/s wr, 15 op/s; 19/489 objects degraded (3.885%) 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[97022]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:41 smithi145 ceph-mon[89145]: pgmap v265: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.7 MiB/s wr, 15 op/s; 19/489 objects degraded (3.885%) 2024-09-13T14:25:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:41 smithi145 ceph-mon[89145]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:25:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:42.983 INFO:tasks.rados.rados.0.smithi026.stdout:2059: done (1 left) 2024-09-13T14:25:42.983 INFO:tasks.rados.rados.0.smithi026.stdout:2060: snap_create 2024-09-13T14:25:42.983 INFO:tasks.rados.rados.0.smithi026.stdout:2058: expect (ObjNum 599 snap 171 seq_num 599) 2024-09-13T14:25:43.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:42 smithi026 ceph-mon[97022]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:25:43.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:42 smithi026 ceph-mon[99461]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:25:43.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:42 smithi145 ceph-mon[89145]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:25:43.989 INFO:tasks.rados.rados.0.smithi026.stdout:2058: done (1 left) 2024-09-13T14:25:43.989 INFO:tasks.rados.rados.0.smithi026.stdout:2060: done (0 left) 2024-09-13T14:25:43.989 INFO:tasks.rados.rados.0.smithi026.stdout:2061: snap_remove snap 171 2024-09-13T14:25:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[99461]: pgmap v268: 121 pgs: 3 peering, 16 active+undersized, 1 active+undersized+degraded, 101 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 30 op/s; 2/453 objects degraded (0.442%); 5.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:25:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[99461]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:25:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[97022]: pgmap v268: 121 pgs: 3 peering, 16 active+undersized, 1 active+undersized+degraded, 101 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 30 op/s; 2/453 objects degraded (0.442%); 5.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:25:44.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:43 smithi026 ceph-mon[97022]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:25:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:43 smithi145 ceph-mon[89145]: pgmap v268: 121 pgs: 3 peering, 16 active+undersized, 1 active+undersized+degraded, 101 active+clean; 222 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 30 op/s; 2/453 objects degraded (0.442%); 5.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:43 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:25:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:43 smithi145 ceph-mon[89145]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:25:44.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:44.997 INFO:tasks.rados.rados.0.smithi026.stdout:2061: done (0 left) 2024-09-13T14:25:44.997 INFO:tasks.rados.rados.0.smithi026.stdout:2062: rollback oid 17 current snap is 190 2024-09-13T14:25:44.997 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 185 2024-09-13T14:25:44.998 INFO:tasks.rados.rados.0.smithi026.stdout:2063: snap_remove snap 172 2024-09-13T14:25:45.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:44 smithi026 ceph-mon[99461]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:25:45.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:44 smithi026 ceph-mon[97022]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:25:45.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:44 smithi145 ceph-mon[89145]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:25:45.993 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:45.993 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2063: done (1 left) 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2064: snap_create 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2062: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2062: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2062: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:25:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 821 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:25:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[97022]: pgmap v271: 121 pgs: 3 peering, 118 active+clean; 232 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 6.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded) 2024-09-13T14:25:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:25:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[97022]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:25:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[99461]: pgmap v271: 121 pgs: 3 peering, 118 active+clean; 232 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 6.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded) 2024-09-13T14:25:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:25:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:45 smithi026 ceph-mon[99461]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:25:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-mon[89145]: pgmap v271: 121 pgs: 3 peering, 118 active+clean; 232 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 6.8 MiB/s, 1 objects/s recovering 2024-09-13T14:25:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/453 objects degraded (0.442%), 1 pg degraded) 2024-09-13T14:25:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:25:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:45 smithi145 ceph-mon[89145]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:25:47.008 INFO:tasks.rados.rados.0.smithi026.stdout:2062: done (1 left) 2024-09-13T14:25:47.008 INFO:tasks.rados.rados.0.smithi026.stdout:2064: done (0 left) 2024-09-13T14:25:47.008 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 191 2024-09-13T14:25:47.008 INFO:tasks.rados.rados.0.smithi026.stdout:2065: seq_num 676 ranges {3776512=524288,4300800=589824,4890624=450560,5341184=163840} 2024-09-13T14:25:47.029 INFO:tasks.rados.rados.0.smithi026.stdout:2065: writing smithi02683976-48 from 3776512 to 4300800 tid 1 2024-09-13T14:25:47.033 INFO:tasks.rados.rados.0.smithi026.stdout:2065: writing smithi02683976-48 from 4300800 to 4890624 tid 2 2024-09-13T14:25:47.035 INFO:tasks.rados.rados.0.smithi026.stdout:2065: writing smithi02683976-48 from 4890624 to 5341184 tid 3 2024-09-13T14:25:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2065: writing smithi02683976-48 from 5341184 to 5505024 tid 4 2024-09-13T14:25:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2066: setattr oid 21 current snap is 191 2024-09-13T14:25:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 1 to smithi02683976-48 2024-09-13T14:25:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 2 to smithi02683976-48 2024-09-13T14:25:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2067: snap_create 2024-09-13T14:25:47.039 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 3 to smithi02683976-48 2024-09-13T14:25:47.039 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 4 to smithi02683976-48 2024-09-13T14:25:47.040 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 5 to smithi02683976-48 2024-09-13T14:25:47.041 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1047 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:25:47.042 INFO:tasks.rados.rados.0.smithi026.stdout:2065: finishing write tid 6 to smithi02683976-48 2024-09-13T14:25:47.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 749 (ObjNum 676 snap 191 seq_num 676) dirty exists 2024-09-13T14:25:47.042 INFO:tasks.rados.rados.0.smithi026.stdout:2065: left oid 48 (ObjNum 676 snap 191 seq_num 676) 2024-09-13T14:25:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:47 smithi145 ceph-mon[89145]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:25:47.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:47.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:47 smithi026 ceph-mon[99461]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:25:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:47 smithi026 ceph-mon[97022]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:25:48.010 INFO:tasks.rados.rados.0.smithi026.stdout:2065: done (2 left) 2024-09-13T14:25:48.011 INFO:tasks.rados.rados.0.smithi026.stdout:2066: done (1 left) 2024-09-13T14:25:48.011 INFO:tasks.rados.rados.0.smithi026.stdout:2067: done (0 left) 2024-09-13T14:25:48.011 INFO:tasks.rados.rados.0.smithi026.stdout:2068: setattr oid 1 current snap is 192 2024-09-13T14:25:48.011 INFO:tasks.rados.rados.0.smithi026.stdout:2069: rmattr oid 4 current snap is 192 2024-09-13T14:25:48.012 INFO:tasks.rados.rados.0.smithi026.stdout:2069: done (1 left) 2024-09-13T14:25:48.012 INFO:tasks.rados.rados.0.smithi026.stdout:2070: copy_from oid 15 from oid 24 current snap is 192 2024-09-13T14:25:48.012 INFO:tasks.rados.rados.0.smithi026.stdout:2071: snap_remove snap 189 2024-09-13T14:25:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:48 smithi026 ceph-mon[97022]: pgmap v274: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.6 MiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:48 smithi026 ceph-mon[97022]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:25:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:48 smithi026 ceph-mon[99461]: pgmap v274: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.6 MiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:48 smithi026 ceph-mon[99461]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:25:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:48 smithi145 ceph-mon[89145]: pgmap v274: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.6 MiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:48 smithi145 ceph-mon[89145]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:2071: done (2 left) 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:2072: setattr oid 39 current snap is 192 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1052 (ObjNum 674 snap 189 seq_num 674) dirty exists 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:2070: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:2070: finishing copy_from to smithi02683976-15 2024-09-13T14:25:48.931 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 666 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:2068: done (2 left) 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:2070: done (1 left) 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:2073: read oid 46 snap 188 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:2073: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 192 2024-09-13T14:25:48.932 INFO:tasks.rados.rados.0.smithi026.stdout:2074: seq_num 677 ranges {2162688=540672,2703360=516096,3219456=655360,3874816=32768} 2024-09-13T14:25:48.947 INFO:tasks.rados.rados.0.smithi026.stdout:2074: writing smithi02683976-22 from 2162688 to 2703360 tid 1 2024-09-13T14:25:48.950 INFO:tasks.rados.rados.0.smithi026.stdout:2074: writing smithi02683976-22 from 2703360 to 3219456 tid 2 2024-09-13T14:25:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:2074: writing smithi02683976-22 from 3219456 to 3874816 tid 3 2024-09-13T14:25:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:2074: writing smithi02683976-22 from 3874816 to 3907584 tid 4 2024-09-13T14:25:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:2075: copy_from oid 27 from oid 33 current snap is 192 2024-09-13T14:25:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 691 (ObjNum 662 snap 188 seq_num 662) dirty exists 2024-09-13T14:25:48.956 INFO:tasks.rados.rados.0.smithi026.stdout:2072: done (3 left) 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:2076: rollback oid 39 current snap is 192 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 190 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:2077: read oid 6 snap -1 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:2077: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:2078: copy_from oid 20 from oid 30 current snap is 192 2024-09-13T14:25:48.957 INFO:tasks.rados.rados.0.smithi026.stdout:2079: copy_from oid 25 from oid 13 current snap is 192 2024-09-13T14:25:48.958 INFO:tasks.rados.rados.0.smithi026.stdout:2080: snap_remove snap 177 2024-09-13T14:25:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:49 smithi026 ceph-mon[97022]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:25:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:49 smithi026 ceph-mon[97022]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:25:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:49 smithi026 ceph-mon[99461]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:25:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:49 smithi026 ceph-mon[99461]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:25:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:49 smithi145 ceph-mon[89145]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:25:49.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:49 smithi145 ceph-mon[89145]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:25:49.931 INFO:tasks.rados.rados.0.smithi026.stdout:2080: done (7 left) 2024-09-13T14:25:49.932 INFO:tasks.rados.rados.0.smithi026.stdout:2081: read oid 16 snap -1 2024-09-13T14:25:49.932 INFO:tasks.rados.rados.0.smithi026.stdout:2081: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:25:49.932 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 192 2024-09-13T14:25:49.932 INFO:tasks.rados.rados.0.smithi026.stdout:2082: seq_num 678 ranges {6905856=565248,7471104=802816,8273920=401408} 2024-09-13T14:25:49.974 INFO:tasks.rados.rados.0.smithi026.stdout:2082: writing smithi02683976-23 from 6905856 to 7471104 tid 1 2024-09-13T14:25:49.980 INFO:tasks.rados.rados.0.smithi026.stdout:2082: writing smithi02683976-23 from 7471104 to 8273920 tid 2 2024-09-13T14:25:49.983 INFO:tasks.rados.rados.0.smithi026.stdout:2082: writing smithi02683976-23 from 8273920 to 8675328 tid 3 2024-09-13T14:25:49.983 INFO:tasks.rados.rados.0.smithi026.stdout:2083: read oid 32 snap 191 2024-09-13T14:25:49.983 INFO:tasks.rados.rados.0.smithi026.stdout:2083: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:25:49.983 INFO:tasks.rados.rados.0.smithi026.stdout:2084: setattr oid 12 current snap is 192 2024-09-13T14:25:49.984 INFO:tasks.rados.rados.0.smithi026.stdout:2073: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:50.140 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 1 to smithi02683976-22 2024-09-13T14:25:50.140 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 2 to smithi02683976-22 2024-09-13T14:25:50.140 INFO:tasks.rados.rados.0.smithi026.stdout:2076: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 3 to smithi02683976-22 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2076: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 4 to smithi02683976-22 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2076: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 694 (ObjNum 662 snap 188 seq_num 662) dirty exists 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 5 to smithi02683976-22 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2075: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2074: finishing write tid 6 to smithi02683976-22 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 962 (ObjNum 677 snap 192 seq_num 677) dirty exists 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2074: left oid 22 (ObjNum 677 snap 192 seq_num 677) 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2075: finishing copy_from to smithi02683976-27 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1012 (ObjNum 480 snap 131 seq_num 480) dirty exists 2024-09-13T14:25:50.141 INFO:tasks.rados.rados.0.smithi026.stdout:2079: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:2079: finishing copy_from to smithi02683976-25 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 966 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:2078: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:2078: finishing copy_from to smithi02683976-20 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 624 (ObjNum 669 snap 188 seq_num 669) dirty exists 2024-09-13T14:25:50.142 INFO:tasks.rados.rados.0.smithi026.stdout:2077: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:50.409 INFO:tasks.rados.rados.0.smithi026.stdout:2081: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:25:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:50 smithi026 ceph-mon[97022]: pgmap v277: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s; 3.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:50 smithi026 ceph-mon[97022]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:25:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:50 smithi026 ceph-mon[99461]: pgmap v277: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s; 3.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:50 smithi026 ceph-mon[99461]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:25:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:50 smithi145 ceph-mon[89145]: pgmap v277: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s; 3.4 MiB/s, 1 objects/s recovering 2024-09-13T14:25:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:50 smithi145 ceph-mon[89145]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:25:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:50.847 INFO:tasks.rados.rados.0.smithi026.stdout:2082: finishing write tid 1 to smithi02683976-23 2024-09-13T14:25:50.847 INFO:tasks.rados.rados.0.smithi026.stdout:2082: finishing write tid 2 to smithi02683976-23 2024-09-13T14:25:50.847 INFO:tasks.rados.rados.0.smithi026.stdout:2082: finishing write tid 3 to smithi02683976-23 2024-09-13T14:25:50.847 INFO:tasks.rados.rados.0.smithi026.stdout:2082: finishing write tid 4 to smithi02683976-23 2024-09-13T14:25:50.848 INFO:tasks.rados.rados.0.smithi026.stdout:2082: finishing write tid 5 to smithi02683976-23 2024-09-13T14:25:50.848 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 973 (ObjNum 678 snap 192 seq_num 678) dirty exists 2024-09-13T14:25:50.848 INFO:tasks.rados.rados.0.smithi026.stdout:2082: left oid 23 (ObjNum 678 snap 192 seq_num 678) 2024-09-13T14:25:50.848 INFO:tasks.rados.rados.0.smithi026.stdout:2083: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:25:50.932 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 963 (ObjNum 617 snap 176 seq_num 617) dirty exists 2024-09-13T14:25:50.932 INFO:tasks.rados.rados.0.smithi026.stdout:2073: done (10 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2074: done (9 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2075: done (8 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2076: done (7 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2077: done (6 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2078: done (5 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2079: done (4 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2081: done (3 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2082: done (2 left) 2024-09-13T14:25:50.933 INFO:tasks.rados.rados.0.smithi026.stdout:2083: done (1 left) 2024-09-13T14:25:50.934 INFO:tasks.rados.rados.0.smithi026.stdout:2084: done (0 left) 2024-09-13T14:25:50.934 INFO:tasks.rados.rados.0.smithi026.stdout:2085: read oid 50 snap -1 2024-09-13T14:25:50.934 INFO:tasks.rados.rados.0.smithi026.stdout:2085: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:25:50.934 INFO:tasks.rados.rados.0.smithi026.stdout:2086: snap_remove snap 188 2024-09-13T14:25:51.022 INFO:tasks.rados.rados.0.smithi026.stdout:2086: done (1 left) 2024-09-13T14:25:51.022 INFO:tasks.rados.rados.0.smithi026.stdout:2087: rollback oid 11 current snap is 192 2024-09-13T14:25:51.022 INFO:tasks.rados.rados.0.smithi026.stdout:2085: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 183 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:2085: done (1 left) 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:2088: read oid 34 snap -1 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:2088: expect (ObjNum 672 snap 189 seq_num 672) 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 192 2024-09-13T14:25:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:2089: seq_num 679 ranges {4038656=581632,4620288=688128,5308416=516096} 2024-09-13T14:25:51.087 INFO:tasks.rados.rados.0.smithi026.stdout:2089: writing smithi02683976-8 from 4038656 to 4620288 tid 1 2024-09-13T14:25:51.092 INFO:tasks.rados.rados.0.smithi026.stdout:2089: writing smithi02683976-8 from 4620288 to 5308416 tid 2 2024-09-13T14:25:51.094 INFO:tasks.rados.rados.0.smithi026.stdout:2089: writing smithi02683976-8 from 5308416 to 5824512 tid 3 2024-09-13T14:25:51.094 INFO:tasks.rados.rados.0.smithi026.stdout:2090: read oid 6 snap -1 2024-09-13T14:25:51.095 INFO:tasks.rados.rados.0.smithi026.stdout:2090: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:51.095 INFO:tasks.rados.rados.0.smithi026.stdout:2091: read oid 36 snap 183 2024-09-13T14:25:51.095 INFO:tasks.rados.rados.0.smithi026.stdout:2091: expect (ObjNum 520130976 snap 21981 seq_num 518850464) 2024-09-13T14:25:51.095 INFO:tasks.rados.rados.0.smithi026.stdout:2092: snap_remove snap 191 2024-09-13T14:25:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:25:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:25:51] "GET /metrics HTTP/1.1" 200 36095 "" "Prometheus/2.51.0" 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:2092: done (5 left) 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:2093: read oid 21 snap 190 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:2093: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 192 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:2087: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 990 (ObjNum 33 snap 0 seq_num 519225088) dirty dne 2024-09-13T14:25:52.029 INFO:tasks.rados.rados.0.smithi026.stdout:2094: seq_num 680 ranges {10051584=598016,10649600=573440,11223040=434176,11657216=196608} 2024-09-13T14:25:52.087 INFO:tasks.rados.rados.0.smithi026.stdout:2094: writing smithi02683976-7 from 10051584 to 10649600 tid 1 2024-09-13T14:25:52.091 INFO:tasks.rados.rados.0.smithi026.stdout:2094: writing smithi02683976-7 from 10649600 to 11223040 tid 2 2024-09-13T14:25:52.094 INFO:tasks.rados.rados.0.smithi026.stdout:2094: writing smithi02683976-7 from 11223040 to 11657216 tid 3 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2094: writing smithi02683976-7 from 11657216 to 11853824 tid 4 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2087: done (6 left) 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2095: snap_create 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2089: finishing write tid 1 to smithi02683976-8 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2089: finishing write tid 2 to smithi02683976-8 2024-09-13T14:25:52.095 INFO:tasks.rados.rados.0.smithi026.stdout:2091: expect (ObjNum 520130976 snap 21981 seq_num 518850464) 2024-09-13T14:25:52.098 INFO:tasks.rados.rados.0.smithi026.stdout:2089: finishing write tid 3 to smithi02683976-8 2024-09-13T14:25:52.098 INFO:tasks.rados.rados.0.smithi026.stdout:2089: finishing write tid 4 to smithi02683976-8 2024-09-13T14:25:52.098 INFO:tasks.rados.rados.0.smithi026.stdout:2089: finishing write tid 5 to smithi02683976-8 2024-09-13T14:25:52.098 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 600 (ObjNum 679 snap 192 seq_num 679) dirty exists 2024-09-13T14:25:52.099 INFO:tasks.rados.rados.0.smithi026.stdout:2089: left oid 8 (ObjNum 679 snap 192 seq_num 679) 2024-09-13T14:25:52.099 INFO:tasks.rados.rados.0.smithi026.stdout:2088: expect (ObjNum 672 snap 189 seq_num 672) 2024-09-13T14:25:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[97022]: pgmap v280: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 628 KiB/s wr, 6 op/s 2024-09-13T14:25:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[97022]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:25:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[99461]: pgmap v280: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 628 KiB/s wr, 6 op/s 2024-09-13T14:25:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[99461]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:25:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:52 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:52 smithi145 ceph-mon[89145]: pgmap v280: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 628 KiB/s wr, 6 op/s 2024-09-13T14:25:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:52 smithi145 ceph-mon[89145]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:25:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:52 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:53.094 INFO:tasks.rados.rados.0.smithi026.stdout:2090: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:53.364 INFO:tasks.rados.rados.0.smithi026.stdout:2093: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:25:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:53 smithi026 ceph-mon[97022]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:25:53.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:53 smithi026 ceph-mon[99461]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:25:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:53 smithi145 ceph-mon[89145]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 1 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 2 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 3 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 4 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 5 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: finishing write tid 6 to smithi02683976-7 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 823 (ObjNum 680 snap 192 seq_num 680) dirty exists 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2094: left oid 7 (ObjNum 680 snap 192 seq_num 680) 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2088: done (6 left) 2024-09-13T14:25:53.516 INFO:tasks.rados.rados.0.smithi026.stdout:2089: done (5 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2090: done (4 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2091: done (3 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2093: done (2 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2094: done (1 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2095: done (0 left) 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 193 2024-09-13T14:25:53.517 INFO:tasks.rados.rados.0.smithi026.stdout:2096: seq_num 681 ranges {1196032=614400,1810432=458752,2269184=638976,2908160=106496} 2024-09-13T14:25:53.525 INFO:tasks.rados.rados.0.smithi026.stdout:2096: writing smithi02683976-49 from 1196032 to 1810432 tid 1 2024-09-13T14:25:53.528 INFO:tasks.rados.rados.0.smithi026.stdout:2096: writing smithi02683976-49 from 1810432 to 2269184 tid 2 2024-09-13T14:25:53.532 INFO:tasks.rados.rados.0.smithi026.stdout:2096: writing smithi02683976-49 from 2269184 to 2908160 tid 3 2024-09-13T14:25:53.532 INFO:tasks.rados.rados.0.smithi026.stdout:2096: writing smithi02683976-49 from 2908160 to 3014656 tid 4 2024-09-13T14:25:53.533 INFO:tasks.rados.rados.0.smithi026.stdout:2097: delete oid 40 current snap is 193 2024-09-13T14:25:53.533 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 1 to smithi02683976-49 2024-09-13T14:25:53.533 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 2 to smithi02683976-49 2024-09-13T14:25:53.534 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 3 to smithi02683976-49 2024-09-13T14:25:53.535 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 4 to smithi02683976-49 2024-09-13T14:25:53.535 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 5 to smithi02683976-49 2024-09-13T14:25:53.536 INFO:tasks.rados.rados.0.smithi026.stdout:2097: done (1 left) 2024-09-13T14:25:53.536 INFO:tasks.rados.rados.0.smithi026.stdout:2098: read oid 6 snap 173 2024-09-13T14:25:53.536 INFO:tasks.rados.rados.0.smithi026.stdout:2098: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:53.536 INFO:tasks.rados.rados.0.smithi026.stdout:2099: rollback oid 3 current snap is 193 2024-09-13T14:25:53.536 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 185 2024-09-13T14:25:53.537 INFO:tasks.rados.rados.0.smithi026.stdout:2100: read oid 42 snap -1 2024-09-13T14:25:53.537 INFO:tasks.rados.rados.0.smithi026.stdout:2100: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:53.537 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 193 2024-09-13T14:25:53.537 INFO:tasks.rados.rados.0.smithi026.stdout:2101: seq_num 683 ranges {8388608=647168,9035776=630784,9666560=573440} 2024-09-13T14:25:53.585 INFO:tasks.rados.rados.0.smithi026.stdout:2101: writing smithi02683976-41 from 8388608 to 9035776 tid 1 2024-09-13T14:25:53.589 INFO:tasks.rados.rados.0.smithi026.stdout:2101: writing smithi02683976-41 from 9035776 to 9666560 tid 2 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:2101: writing smithi02683976-41 from 9666560 to 10240000 tid 3 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:2102: read oid 28 snap -1 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:2102: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 193 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:2099: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 0 (ObjNum 129 snap 0 seq_num 518851552) dirty dne 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:2096: finishing write tid 6 to smithi02683976-49 2024-09-13T14:25:53.593 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 936 (ObjNum 681 snap 193 seq_num 681) dirty exists 2024-09-13T14:25:53.594 INFO:tasks.rados.rados.0.smithi026.stdout:2096: left oid 49 (ObjNum 681 snap 193 seq_num 681) 2024-09-13T14:25:53.594 INFO:tasks.rados.rados.0.smithi026.stdout:2100: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:53.673 INFO:tasks.rados.rados.0.smithi026.stdout:2098: expect (ObjNum 603 snap 172 seq_num 603) 2024-09-13T14:25:53.942 INFO:tasks.rados.rados.0.smithi026.stdout:2101: finishing write tid 1 to smithi02683976-41 2024-09-13T14:25:53.942 INFO:tasks.rados.rados.0.smithi026.stdout:2101: finishing write tid 2 to smithi02683976-41 2024-09-13T14:25:53.942 INFO:tasks.rados.rados.0.smithi026.stdout:2101: finishing write tid 3 to smithi02683976-41 2024-09-13T14:25:53.943 INFO:tasks.rados.rados.0.smithi026.stdout:2101: finishing write tid 4 to smithi02683976-41 2024-09-13T14:25:53.943 INFO:tasks.rados.rados.0.smithi026.stdout:2101: finishing write tid 5 to smithi02683976-41 2024-09-13T14:25:53.943 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 825 (ObjNum 683 snap 193 seq_num 683) dirty exists 2024-09-13T14:25:53.943 INFO:tasks.rados.rados.0.smithi026.stdout:2101: left oid 41 (ObjNum 683 snap 193 seq_num 683) 2024-09-13T14:25:53.943 INFO:tasks.rados.rados.0.smithi026.stdout:2102: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:25:54.022 INFO:tasks.rados.rados.0.smithi026.stdout:2103: seq_num 684 ranges {0=663552,663552=507904,1171456=409600,1581056=286720} 2024-09-13T14:25:54.025 INFO:tasks.rados.rados.0.smithi026.stdout:2103: writing smithi02683976-13 from 0 to 663552 tid 1 2024-09-13T14:25:54.028 INFO:tasks.rados.rados.0.smithi026.stdout:2103: writing smithi02683976-13 from 663552 to 1171456 tid 2 2024-09-13T14:25:54.031 INFO:tasks.rados.rados.0.smithi026.stdout:2103: writing smithi02683976-13 from 1171456 to 1581056 tid 3 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2103: writing smithi02683976-13 from 1581056 to 1867776 tid 4 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2096: done (6 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2098: done (5 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2099: done (4 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2100: done (3 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2101: done (2 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2102: done (1 left) 2024-09-13T14:25:54.034 INFO:tasks.rados.rados.0.smithi026.stdout:2104: snap_remove snap 183 2024-09-13T14:25:54.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[97022]: pgmap v283: 121 pgs: 121 active+clean; 246 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.5 MiB/s wr, 32 op/s 2024-09-13T14:25:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[97022]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:25:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[97022]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:25:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[99461]: pgmap v283: 121 pgs: 121 active+clean; 246 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.5 MiB/s wr, 32 op/s 2024-09-13T14:25:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[99461]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:25:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:54 smithi026 ceph-mon[99461]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:25:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:54 smithi145 ceph-mon[89145]: pgmap v283: 121 pgs: 121 active+clean; 246 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.5 MiB/s wr, 32 op/s 2024-09-13T14:25:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:54 smithi145 ceph-mon[89145]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:25:54.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:54 smithi145 ceph-mon[89145]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:25:54.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:55.042 INFO:tasks.rados.rados.0.smithi026.stdout:2104: done (1 left) 2024-09-13T14:25:55.042 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 193 2024-09-13T14:25:55.042 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 1 to smithi02683976-13 2024-09-13T14:25:55.042 INFO:tasks.rados.rados.0.smithi026.stdout:2105: seq_num 685 ranges {0=679936,679936=442368,1122304=622592,1744896=139264} 2024-09-13T14:25:55.046 INFO:tasks.rados.rados.0.smithi026.stdout:2105: writing smithi02683976-25 from 0 to 679936 tid 1 2024-09-13T14:25:55.050 INFO:tasks.rados.rados.0.smithi026.stdout:2105: writing smithi02683976-25 from 679936 to 1122304 tid 2 2024-09-13T14:25:55.055 INFO:tasks.rados.rados.0.smithi026.stdout:2105: writing smithi02683976-25 from 1122304 to 1744896 tid 3 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2105: writing smithi02683976-25 from 1744896 to 1884160 tid 4 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2106: read oid 15 snap -1 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2106: expect (ObjNum 647 snap 182 seq_num 647) 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2107: setattr oid 48 current snap is 193 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 2 to smithi02683976-13 2024-09-13T14:25:55.056 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 3 to smithi02683976-13 2024-09-13T14:25:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 4 to smithi02683976-13 2024-09-13T14:25:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 5 to smithi02683976-13 2024-09-13T14:25:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2103: finishing write tid 6 to smithi02683976-13 2024-09-13T14:25:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1028 (ObjNum 684 snap 193 seq_num 684) dirty exists 2024-09-13T14:25:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2103: left oid 13 (ObjNum 684 snap 193 seq_num 684) 2024-09-13T14:25:55.058 INFO:tasks.rados.rados.0.smithi026.stdout:2103: done (3 left) 2024-09-13T14:25:55.058 INFO:tasks.rados.rados.0.smithi026.stdout:2108: delete oid 33 current snap is 193 2024-09-13T14:25:55.063 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 1 to smithi02683976-25 2024-09-13T14:25:55.065 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 2 to smithi02683976-25 2024-09-13T14:25:55.065 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 3 to smithi02683976-25 2024-09-13T14:25:55.067 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 4 to smithi02683976-25 2024-09-13T14:25:55.067 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 5 to smithi02683976-25 2024-09-13T14:25:55.069 INFO:tasks.rados.rados.0.smithi026.stdout:2108: done (3 left) 2024-09-13T14:25:55.069 INFO:tasks.rados.rados.0.smithi026.stdout:2109: read oid 50 snap -1 2024-09-13T14:25:55.069 INFO:tasks.rados.rados.0.smithi026.stdout:2109: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:25:55.069 INFO:tasks.rados.rados.0.smithi026.stdout:2110: snap_remove snap 173 2024-09-13T14:25:56.044 INFO:tasks.rados.rados.0.smithi026.stdout:2110: done (4 left) 2024-09-13T14:25:56.044 INFO:tasks.rados.rados.0.smithi026.stdout:2111: copy_from oid 45 from oid 7 current snap is 193 2024-09-13T14:25:56.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 750 (ObjNum 676 snap 191 seq_num 676) dirty exists 2024-09-13T14:25:56.045 INFO:tasks.rados.rados.0.smithi026.stdout:2105: finishing write tid 6 to smithi02683976-25 2024-09-13T14:25:56.045 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 971 (ObjNum 685 snap 193 seq_num 685) dirty exists 2024-09-13T14:25:56.045 INFO:tasks.rados.rados.0.smithi026.stdout:2105: left oid 25 (ObjNum 685 snap 193 seq_num 685) 2024-09-13T14:25:56.045 INFO:tasks.rados.rados.0.smithi026.stdout:2109: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:25:56.048 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:56.048 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:55 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:56.095 INFO:tasks.rados.rados.0.smithi026.stdout:2106: expect (ObjNum 647 snap 182 seq_num 647) 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2111: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2111: finishing copy_from to smithi02683976-45 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 823 (ObjNum 680 snap 192 seq_num 680) dirty exists 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2105: done (4 left) 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2106: done (3 left) 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2107: done (2 left) 2024-09-13T14:25:56.395 INFO:tasks.rados.rados.0.smithi026.stdout:2109: done (1 left) 2024-09-13T14:25:56.396 INFO:tasks.rados.rados.0.smithi026.stdout:2111: done (0 left) 2024-09-13T14:25:56.396 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 193 2024-09-13T14:25:56.396 INFO:tasks.rados.rados.0.smithi026.stdout:2112: seq_num 687 ranges {0=712704,712704=614400,1327104=589824} 2024-09-13T14:25:56.398 INFO:tasks.rados.rados.0.smithi026.stdout:2112: writing smithi02683976-35 from 0 to 712704 tid 1 2024-09-13T14:25:56.402 INFO:tasks.rados.rados.0.smithi026.stdout:2112: writing smithi02683976-35 from 712704 to 1327104 tid 2 2024-09-13T14:25:56.407 INFO:tasks.rados.rados.0.smithi026.stdout:2112: writing smithi02683976-35 from 1327104 to 1916928 tid 3 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:2113: rollback oid 1 current snap is 193 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 184 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:2112: finishing write tid 1 to smithi02683976-35 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:2114: copy_from oid 13 from oid 21 current snap is 193 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:2112: finishing write tid 2 to smithi02683976-35 2024-09-13T14:25:56.408 INFO:tasks.rados.rados.0.smithi026.stdout:2115: snap_create 2024-09-13T14:25:56.411 INFO:tasks.rados.rados.0.smithi026.stdout:2112: finishing write tid 3 to smithi02683976-35 2024-09-13T14:25:56.411 INFO:tasks.rados.rados.0.smithi026.stdout:2112: finishing write tid 4 to smithi02683976-35 2024-09-13T14:25:56.411 INFO:tasks.rados.rados.0.smithi026.stdout:2113: finishing rollback tid 0 to smithi02683976-1 2024-09-13T14:25:56.412 INFO:tasks.rados.rados.0.smithi026.stdout:2113: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:25:56.414 INFO:tasks.rados.rados.0.smithi026.stdout:2112: finishing write tid 5 to smithi02683976-35 2024-09-13T14:25:56.414 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1082 (ObjNum 687 snap 193 seq_num 687) dirty exists 2024-09-13T14:25:56.414 INFO:tasks.rados.rados.0.smithi026.stdout:2112: left oid 35 (ObjNum 687 snap 193 seq_num 687) 2024-09-13T14:25:56.414 INFO:tasks.rados.rados.0.smithi026.stdout:2113: finishing rollback tid 2 to smithi02683976-1 2024-09-13T14:25:56.414 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1086 (ObjNum 636 snap 180 seq_num 636) dirty exists 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[99461]: pgmap v286: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[99461]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[99461]: Upgrade: osd.4 is safe to restart 2024-09-13T14:25:56.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[97022]: pgmap v286: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-13T14:25:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[97022]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:25:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:56 smithi026 ceph-mon[97022]: Upgrade: osd.4 is safe to restart 2024-09-13T14:25:56.422 INFO:tasks.rados.rados.0.smithi026.stdout:2114: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:25:56.424 INFO:tasks.rados.rados.0.smithi026.stdout:2114: finishing copy_from to smithi02683976-13 2024-09-13T14:25:56.424 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1047 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:25:56.438 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-mon[89145]: pgmap v286: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-13T14:25:56.438 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-mon[89145]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:25:56.439 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:25:56.439 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:25:56.439 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-mon[89145]: Upgrade: osd.4 is safe to restart 2024-09-13T14:25:57.161 INFO:tasks.rados.rados.0.smithi026.stdout:2112: done (3 left) 2024-09-13T14:25:57.161 INFO:tasks.rados.rados.0.smithi026.stdout:2113: done (2 left) 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2114: done (1 left) 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2115: done (0 left) 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2116: copy_from oid 35 from oid 2 current snap is 194 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2117: copy_from oid 24 from oid 12 current snap is 194 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2118: copy_from oid 44 from oid 36 current snap is 194 2024-09-13T14:25:57.162 INFO:tasks.rados.rados.0.smithi026.stdout:2119: snap_create 2024-09-13T14:25:57.164 INFO:tasks.rados.rados.0.smithi026.stdout:2118: finishing copy_from to smithi02683976-44 2024-09-13T14:25:57.164 INFO:tasks.rados.rados.0.smithi026.stdout:2118: got expected ENOENT (src dne) 2024-09-13T14:25:57.165 INFO:tasks.rados.rados.0.smithi026.stdout:2118: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:25:57.165 INFO:tasks.rados.rados.0.smithi026.stdout:2116: finishing copy_from to smithi02683976-35 2024-09-13T14:25:57.165 INFO:tasks.rados.rados.0.smithi026.stdout:2116: got expected ENOENT (src dne) 2024-09-13T14:25:57.165 INFO:tasks.rados.rados.0.smithi026.stdout:2116: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:25:57.166 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:25:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:25:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:57.166 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:25:57.166 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.166 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.166 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:25:57.167 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:57.170 INFO:tasks.rados.rados.0.smithi026.stdout:2117: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:25:57.172 INFO:tasks.rados.rados.0.smithi026.stdout:2117: finishing copy_from to smithi02683976-24 2024-09-13T14:25:57.172 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 963 (ObjNum 617 snap 176 seq_num 617) dirty exists 2024-09-13T14:25:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:25:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:25:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:25:57.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:2116: done (3 left) 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:2117: done (2 left) 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:2118: done (1 left) 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:2119: done (0 left) 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 195 2024-09-13T14:25:58.169 INFO:tasks.rados.rados.0.smithi026.stdout:2120: seq_num 688 ranges {11853824=729088,12582912=499712,13082624=704512} 2024-09-13T14:25:58.237 INFO:tasks.rados.rados.0.smithi026.stdout:2120: writing smithi02683976-45 from 11853824 to 12582912 tid 1 2024-09-13T14:25:58.240 INFO:tasks.rados.rados.0.smithi026.stdout:2120: writing smithi02683976-45 from 12582912 to 13082624 tid 2 2024-09-13T14:25:58.244 INFO:tasks.rados.rados.0.smithi026.stdout:2120: writing smithi02683976-45 from 13082624 to 13787136 tid 3 2024-09-13T14:25:58.245 INFO:tasks.rados.rados.0.smithi026.stdout:2121: rollback oid 15 current snap is 195 2024-09-13T14:25:58.245 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 194 2024-09-13T14:25:58.245 INFO:tasks.rados.rados.0.smithi026.stdout:2120: finishing write tid 1 to smithi02683976-45 2024-09-13T14:25:58.245 INFO:tasks.rados.rados.0.smithi026.stdout:2120: finishing write tid 2 to smithi02683976-45 2024-09-13T14:25:58.245 INFO:tasks.rados.rados.0.smithi026.stdout:2122: snap_remove snap 185 2024-09-13T14:25:58.351 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:58 smithi145 ceph-mon[89145]: Upgrade: Updating osd.4 2024-09-13T14:25:58.351 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:58 smithi145 ceph-mon[89145]: Deploying daemon osd.4 on smithi145 2024-09-13T14:25:58.352 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:58 smithi145 ceph-mon[89145]: pgmap v289: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:25:58.352 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:58 smithi145 ceph-mon[89145]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:25:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[97022]: Upgrade: Updating osd.4 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[97022]: Deploying daemon osd.4 on smithi145 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[97022]: pgmap v289: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[97022]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[99461]: Upgrade: Updating osd.4 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[99461]: Deploying daemon osd.4 on smithi145 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[99461]: pgmap v289: 121 pgs: 121 active+clean; 248 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:25:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:58 smithi026 ceph-mon[99461]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:25:58.936 INFO:tasks.rados.rados.0.smithi026.stdout:2122: done (2 left) 2024-09-13T14:25:58.936 INFO:tasks.rados.rados.0.smithi026.stdout:2123: read oid 8 snap 190 2024-09-13T14:25:58.936 INFO:tasks.rados.rados.0.smithi026.stdout:2123: expect (ObjNum 664 snap 188 seq_num 664) 2024-09-13T14:25:58.936 INFO:tasks.rados.rados.0.smithi026.stdout:2124: rollback oid 47 current snap is 195 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 190 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2120: finishing write tid 3 to smithi02683976-45 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2121: finishing rollback tid 0 to smithi02683976-15 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2120: finishing write tid 4 to smithi02683976-45 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2121: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2121: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 669 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2120: finishing write tid 5 to smithi02683976-45 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 827 (ObjNum 688 snap 195 seq_num 688) dirty exists 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2120: left oid 45 (ObjNum 688 snap 195 seq_num 688) 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2120: done (3 left) 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2121: done (2 left) 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 195 2024-09-13T14:25:58.937 INFO:tasks.rados.rados.0.smithi026.stdout:2125: seq_num 689 ranges {3014656=745472,3760128=786432,4546560=417792} 2024-09-13T14:25:58.959 INFO:tasks.rados.rados.0.smithi026.stdout:2125: writing smithi02683976-49 from 3014656 to 3760128 tid 1 2024-09-13T14:25:58.965 INFO:tasks.rados.rados.0.smithi026.stdout:2125: writing smithi02683976-49 from 3760128 to 4546560 tid 2 2024-09-13T14:25:58.968 INFO:tasks.rados.rados.0.smithi026.stdout:2125: writing smithi02683976-49 from 4546560 to 4964352 tid 3 2024-09-13T14:25:58.968 INFO:tasks.rados.rados.0.smithi026.stdout:2126: copy_from oid 32 from oid 11 current snap is 195 2024-09-13T14:25:58.969 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 195 2024-09-13T14:25:58.969 INFO:tasks.rados.rados.0.smithi026.stdout:2127: seq_num 690 ranges {0=761856,761856=663552,1425408=409600,1835008=131072} 2024-09-13T14:25:58.974 INFO:tasks.rados.rados.0.smithi026.stdout:2127: writing smithi02683976-36 from 0 to 761856 tid 1 2024-09-13T14:25:58.979 INFO:tasks.rados.rados.0.smithi026.stdout:2127: writing smithi02683976-36 from 761856 to 1425408 tid 2 2024-09-13T14:25:58.982 INFO:tasks.rados.rados.0.smithi026.stdout:2127: writing smithi02683976-36 from 1425408 to 1835008 tid 3 2024-09-13T14:25:58.983 INFO:tasks.rados.rados.0.smithi026.stdout:2127: writing smithi02683976-36 from 1835008 to 1966080 tid 4 2024-09-13T14:25:58.983 INFO:tasks.rados.rados.0.smithi026.stdout:2128: snap_remove snap 184 2024-09-13T14:25:59.383 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:59 smithi145 ceph-mon[89145]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:25:59.383 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:25:59 smithi145 ceph-mon[89145]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:25:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:59 smithi026 ceph-mon[97022]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:25:59.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:25:59 smithi026 ceph-mon[97022]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:25:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:59 smithi026 ceph-mon[99461]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:25:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:25:59 smithi026 ceph-mon[99461]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:25:59.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:25:59 smithi145 systemd[1]: Stopping Ceph osd.4 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:25:59.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:25:59 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[36471]: 2024-09-13T14:25:59.625+0000 7fa376fd4640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:25:59.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:25:59 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[36471]: 2024-09-13T14:25:59.625+0000 7fa376fd4640 -1 osd.4 463 *** Got signal Terminated *** 2024-09-13T14:25:59.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:25:59 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[36471]: 2024-09-13T14:25:59.625+0000 7fa376fd4640 -1 osd.4 463 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:25:59.951 INFO:tasks.rados.rados.0.smithi026.stdout:2128: done (5 left) 2024-09-13T14:25:59.951 INFO:tasks.rados.rados.0.smithi026.stdout:2129: snap_remove snap 194 2024-09-13T14:26:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:00 smithi145 ceph-mon[89145]: pgmap v292: 121 pgs: 121 active+clean; 237 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:26:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:00 smithi145 ceph-mon[89145]: osd.4 marked itself down and dead 2024-09-13T14:26:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:00 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:00 smithi145 ceph-mon[89145]: osdmap e464: 8 total, 7 up, 8 in 2024-09-13T14:26:00.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:00.456 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:00 smithi145 podman[94731]: 2024-09-13 14:26:00.318983914 +0000 UTC m=+0.793275195 container died 74b38ce01808c5d6219794b2b1439f885fae247988f2d4e833bd1270abceff6e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:26:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[97022]: pgmap v292: 121 pgs: 121 active+clean; 237 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[97022]: osd.4 marked itself down and dead 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[97022]: osdmap e464: 8 total, 7 up, 8 in 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[99461]: pgmap v292: 121 pgs: 121 active+clean; 237 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.0 MiB/s wr, 15 op/s 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[99461]: osd.4 marked itself down and dead 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:00 smithi026 ceph-mon[99461]: osdmap e464: 8 total, 7 up, 8 in 2024-09-13T14:26:00.942 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:00 smithi145 podman[94731]: 2024-09-13 14:26:00.706995955 +0000 UTC m=+1.181287238 container cleanup 74b38ce01808c5d6219794b2b1439f885fae247988f2d4e833bd1270abceff6e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T14:26:00.942 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:00 smithi145 bash[94731]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T14:26:00.946 INFO:tasks.rados.rados.0.smithi026.stdout:2129: done (5 left) 2024-09-13T14:26:00.947 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 195 2024-09-13T14:26:00.947 INFO:tasks.rados.rados.0.smithi026.stdout:2130: seq_num 691 ranges {3907584=778240,4685824=548864,5234688=614400,5849088=40960} 2024-09-13T14:26:00.973 INFO:tasks.rados.rados.0.smithi026.stdout:2130: writing smithi02683976-22 from 3907584 to 4685824 tid 1 2024-09-13T14:26:00.976 INFO:tasks.rados.rados.0.smithi026.stdout:2130: writing smithi02683976-22 from 4685824 to 5234688 tid 2 2024-09-13T14:26:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2130: writing smithi02683976-22 from 5234688 to 5849088 tid 3 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2130: writing smithi02683976-22 from 5849088 to 5890048 tid 4 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2131: rmattr oid 25 current snap is 195 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2124: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2124: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2124: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 753 (ObjNum 668 snap 188 seq_num 668) dirty exists 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2124: done (6 left) 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2132: rollback oid 4 current snap is 195 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 192 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2125: finishing write tid 1 to smithi02683976-49 2024-09-13T14:26:00.981 INFO:tasks.rados.rados.0.smithi026.stdout:2126: finishing copy_from to smithi02683976-32 2024-09-13T14:26:00.982 INFO:tasks.rados.rados.0.smithi026.stdout:2126: got expected ENOENT (src dne) 2024-09-13T14:26:00.982 INFO:tasks.rados.rados.0.smithi026.stdout:2126: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:26:00.982 INFO:tasks.rados.rados.0.smithi026.stdout:2125: finishing write tid 2 to smithi02683976-49 2024-09-13T14:26:00.982 INFO:tasks.rados.rados.0.smithi026.stdout:2125: finishing write tid 3 to smithi02683976-49 2024-09-13T14:26:00.982 INFO:tasks.rados.rados.0.smithi026.stdout:2123: expect (ObjNum 664 snap 188 seq_num 664) 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:2125: finishing write tid 4 to smithi02683976-49 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:2125: finishing write tid 5 to smithi02683976-49 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 940 (ObjNum 689 snap 195 seq_num 689) dirty exists 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:2125: left oid 49 (ObjNum 689 snap 195 seq_num 689) 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 1 to smithi02683976-36 2024-09-13T14:26:01.151 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 2 to smithi02683976-36 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 3 to smithi02683976-36 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 4 to smithi02683976-36 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 5 to smithi02683976-36 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2127: finishing write tid 6 to smithi02683976-36 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1099 (ObjNum 690 snap 195 seq_num 690) dirty exists 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2127: left oid 36 (ObjNum 690 snap 195 seq_num 690) 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 1 to smithi02683976-22 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 2 to smithi02683976-22 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 972 (ObjNum 685 snap 193 seq_num 685) dirty exists 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2132: finishing rollback tid 0 to smithi02683976-4 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 3 to smithi02683976-22 2024-09-13T14:26:01.152 INFO:tasks.rados.rados.0.smithi026.stdout:2132: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2132: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 672 (ObjNum 364 snap 102 seq_num 364) dirty exists 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 4 to smithi02683976-22 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 5 to smithi02683976-22 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2123: done (6 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2125: done (5 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2126: done (4 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2127: done (3 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2131: done (2 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2132: done (1 left) 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2133: read oid 2 snap -1 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2133: expect deleted 2024-09-13T14:26:01.153 INFO:tasks.rados.rados.0.smithi026.stdout:2134: read oid 43 snap 192 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2134: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2130: finishing write tid 6 to smithi02683976-22 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 968 (ObjNum 691 snap 195 seq_num 691) dirty exists 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2130: left oid 22 (ObjNum 691 snap 195 seq_num 691) 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2135: read oid 8 snap -1 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2135: expect (ObjNum 679 snap 192 seq_num 679) 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2130: done (3 left) 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2136: read oid 18 snap -1 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2136: expect deleted 2024-09-13T14:26:01.154 INFO:tasks.rados.rados.0.smithi026.stdout:2137: delete oid 27 current snap is 195 2024-09-13T14:26:01.155 INFO:tasks.rados.rados.0.smithi026.stdout:2133: done (4 left) 2024-09-13T14:26:01.155 INFO:tasks.rados.rados.0.smithi026.stdout:2136: done (3 left) 2024-09-13T14:26:01.155 INFO:tasks.rados.rados.0.smithi026.stdout:2137: done (2 left) 2024-09-13T14:26:01.156 INFO:tasks.rados.rados.0.smithi026.stdout:2138: read oid 7 snap -1 2024-09-13T14:26:01.156 INFO:tasks.rados.rados.0.smithi026.stdout:2138: expect (ObjNum 680 snap 192 seq_num 680) 2024-09-13T14:26:01.156 INFO:tasks.rados.rados.0.smithi026.stdout:2139: snap_remove snap 193 2024-09-13T14:26:01.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:00 smithi145 podman[94755]: 2024-09-13 14:26:00.941769056 +0000 UTC m=+0.626027590 container remove 74b38ce01808c5d6219794b2b1439f885fae247988f2d4e833bd1270abceff6e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T14:26:01.596 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.181022027 +0000 UTC m=+0.016245396 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:01.596 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.301135952 +0000 UTC m=+0.136359310 container create 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:26:01.596 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.520338391 +0000 UTC m=+0.355561750 container init 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:26:01.596 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.525191913 +0000 UTC m=+0.360415275 container start 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2) 2024-09-13T14:26:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:01] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-13T14:26:01.944 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.595110058 +0000 UTC m=+0.430333427 container attach 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:26:01.944 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:01 smithi145 podman[94846]: 2024-09-13 14:26:01.713805925 +0000 UTC m=+0.549029284 container died 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:26:01.951 INFO:tasks.rados.rados.0.smithi026.stdout:2139: done (3 left) 2024-09-13T14:26:01.951 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 195 2024-09-13T14:26:01.951 INFO:tasks.rados.rados.0.smithi026.stdout:2140: seq_num 693 ranges {2555904=409600,2965504=770048,3735552=581632,4317184=253952} 2024-09-13T14:26:01.968 INFO:tasks.rados.rados.0.smithi026.stdout:2140: writing smithi02683976-47 from 2555904 to 2965504 tid 1 2024-09-13T14:26:01.972 INFO:tasks.rados.rados.0.smithi026.stdout:2140: writing smithi02683976-47 from 2965504 to 3735552 tid 2 2024-09-13T14:26:01.976 INFO:tasks.rados.rados.0.smithi026.stdout:2140: writing smithi02683976-47 from 3735552 to 4317184 tid 3 2024-09-13T14:26:01.978 INFO:tasks.rados.rados.0.smithi026.stdout:2140: writing smithi02683976-47 from 4317184 to 4571136 tid 4 2024-09-13T14:26:01.978 INFO:tasks.rados.rados.0.smithi026.stdout:2141: copy_from oid 49 from oid 48 current snap is 195 2024-09-13T14:26:01.978 INFO:tasks.rados.rados.0.smithi026.stdout:2134: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:26:02.046 INFO:tasks.rados.rados.0.smithi026.stdout:2135: expect (ObjNum 679 snap 192 seq_num 679) 2024-09-13T14:26:02.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:01 smithi145 ceph-mon[89145]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 7 stale+active+clean, 106 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-13T14:26:02.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:01 smithi145 ceph-mon[89145]: osdmap e465: 8 total, 7 up, 8 in 2024-09-13T14:26:02.328 INFO:tasks.rados.rados.0.smithi026.stdout:2138: expect (ObjNum 680 snap 192 seq_num 680) 2024-09-13T14:26:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:01 smithi026 ceph-mon[97022]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 7 stale+active+clean, 106 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-13T14:26:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:01 smithi026 ceph-mon[97022]: osdmap e465: 8 total, 7 up, 8 in 2024-09-13T14:26:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:01 smithi026 ceph-mon[99461]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 7 stale+active+clean, 106 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-13T14:26:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:01 smithi026 ceph-mon[99461]: osdmap e465: 8 total, 7 up, 8 in 2024-09-13T14:26:02.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 podman[94889]: 2024-09-13 14:26:02.440625742 +0000 UTC m=+0.718228921 container remove 679545d717b73e66737d2ded55a0578c6ef5be40461c6815d21f3d69b7dba6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:26:02.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T14:26:02.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 systemd[1]: Stopped Ceph osd.4 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:26:02.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.4.service: Consumed 14.794s CPU time. 2024-09-13T14:26:03.130 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:02 smithi145 ceph-mon[89145]: osdmap e466: 8 total, 7 up, 8 in 2024-09-13T14:26:03.131 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 systemd[1]: Starting Ceph osd.4 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:26:03.131 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:02 smithi145 podman[94993]: 2024-09-13 14:26:02.898260044 +0000 UTC m=+0.016028762 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:03.131 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 podman[94993]: 2024-09-13 14:26:03.017149001 +0000 UTC m=+0.134917719 container create d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:26:03.386 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 podman[94993]: 2024-09-13 14:26:03.236657757 +0000 UTC m=+0.354426483 container init d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T14:26:03.386 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 podman[94993]: 2024-09-13 14:26:03.241632568 +0000 UTC m=+0.359401287 container start d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:26:03.386 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 podman[94993]: 2024-09-13 14:26:03.327825429 +0000 UTC m=+0.445594148 container attach d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T14:26:03.387 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.387 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.387 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.387 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:02 smithi026 ceph-mon[99461]: osdmap e466: 8 total, 7 up, 8 in 2024-09-13T14:26:03.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:02 smithi026 ceph-mon[97022]: osdmap e466: 8 total, 7 up, 8 in 2024-09-13T14:26:03.536 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 1 to smithi02683976-47 2024-09-13T14:26:03.536 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 2 to smithi02683976-47 2024-09-13T14:26:03.536 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 3 to smithi02683976-47 2024-09-13T14:26:03.536 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 4 to smithi02683976-47 2024-09-13T14:26:03.536 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 5 to smithi02683976-47 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2140: finishing write tid 6 to smithi02683976-47 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 758 (ObjNum 693 snap 195 seq_num 693) dirty exists 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2140: left oid 47 (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2141: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2141: finishing copy_from to smithi02683976-49 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 750 (ObjNum 676 snap 191 seq_num 676) dirty exists 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2134: done (4 left) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2135: done (3 left) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2138: done (2 left) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2140: done (1 left) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2141: done (0 left) 2024-09-13T14:26:03.537 INFO:tasks.rados.rados.0.smithi026.stdout:2142: snap_create 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:26:03.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-13T14:26:03.707 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-13T14:26:03.936 INFO:tasks.rados.rados.0.smithi026.stdout:2142: done (0 left) 2024-09-13T14:26:03.936 INFO:tasks.rados.rados.0.smithi026.stdout:2143: copy_from oid 24 from oid 14 current snap is 196 2024-09-13T14:26:03.936 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 196 2024-09-13T14:26:03.936 INFO:tasks.rados.rados.0.smithi026.stdout:2144: seq_num 694 ranges {434176=425984,860160=655360,1515520=786432,2301952=163840} 2024-09-13T14:26:03.941 INFO:tasks.rados.rados.0.smithi026.stdout:2144: writing smithi02683976-10 from 434176 to 860160 tid 1 2024-09-13T14:26:03.945 INFO:tasks.rados.rados.0.smithi026.stdout:2144: writing smithi02683976-10 from 860160 to 1515520 tid 2 2024-09-13T14:26:03.949 INFO:tasks.rados.rados.0.smithi026.stdout:2144: writing smithi02683976-10 from 1515520 to 2301952 tid 3 2024-09-13T14:26:03.950 INFO:tasks.rados.rados.0.smithi026.stdout:2144: writing smithi02683976-10 from 2301952 to 2465792 tid 4 2024-09-13T14:26:03.951 INFO:tasks.rados.rados.0.smithi026.stdout:2145: snap_remove snap 192 2024-09-13T14:26:04.089 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-mon[89145]: pgmap v298: 121 pgs: 24 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 1 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.6 MiB/s wr, 18 op/s; 37/333 objects degraded (11.111%) 2024-09-13T14:26:04.089 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 37/333 objects degraded (11.111%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:04.090 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-mon[89145]: osdmap e467: 8 total, 7 up, 8 in 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:26:04.090 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate[95008]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-13T14:26:04.091 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 bash[94993]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-13T14:26:04.091 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:03 smithi145 podman[94993]: 2024-09-13 14:26:03.78927577 +0000 UTC m=+0.907044492 container died d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:26:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[97022]: pgmap v298: 121 pgs: 24 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 1 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.6 MiB/s wr, 18 op/s; 37/333 objects degraded (11.111%) 2024-09-13T14:26:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 37/333 objects degraded (11.111%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[97022]: osdmap e467: 8 total, 7 up, 8 in 2024-09-13T14:26:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[99461]: pgmap v298: 121 pgs: 24 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 1 stale+active+clean, 11 active+undersized+degraded, 77 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 3.6 MiB/s wr, 18 op/s; 37/333 objects degraded (11.111%) 2024-09-13T14:26:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 37/333 objects degraded (11.111%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:03 smithi026 ceph-mon[99461]: osdmap e467: 8 total, 7 up, 8 in 2024-09-13T14:26:04.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:04.797 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 podman[95148]: 2024-09-13 14:26:04.456893746 +0000 UTC m=+0.657897282 container remove d55a5940abd62591f49924a61ce2687230c6e823de4129da4dff38fa7380507c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux ) 2024-09-13T14:26:04.797 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 podman[95215]: 2024-09-13 14:26:04.666120244 +0000 UTC m=+0.111930353 container create 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:26:04.797 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 podman[95215]: 2024-09-13 14:26:04.570467618 +0000 UTC m=+0.016277730 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2145: done (2 left) 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2146: read oid 36 snap -1 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2146: expect (ObjNum 690 snap 195 seq_num 690) 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2147: delete oid 40 current snap is 196 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 1 to smithi02683976-10 2024-09-13T14:26:04.937 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 2 to smithi02683976-10 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 3 to smithi02683976-10 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 4 to smithi02683976-10 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 5 to smithi02683976-10 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2144: finishing write tid 6 to smithi02683976-10 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 945 (ObjNum 694 snap 196 seq_num 694) dirty exists 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2144: left oid 10 (ObjNum 694 snap 196 seq_num 694) 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2143: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:2143: finishing copy_from to smithi02683976-24 2024-09-13T14:26:04.938 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 844 (ObjNum 616 snap 176 seq_num 616) dirty exists 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2143: done (3 left) 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2144: done (2 left) 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2147: done (1 left) 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2148: rmattr oid 25 current snap is 196 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2149: rollback oid 24 current snap is 196 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 190 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2150: rmattr oid 23 current snap is 196 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:2151: rollback oid 37 current snap is 196 2024-09-13T14:26:04.944 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 195 2024-09-13T14:26:04.945 INFO:tasks.rados.rados.0.smithi026.stdout:2152: read oid 9 snap -1 2024-09-13T14:26:04.945 INFO:tasks.rados.rados.0.smithi026.stdout:2152: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:04.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 196 2024-09-13T14:26:04.945 INFO:tasks.rados.rados.0.smithi026.stdout:2153: seq_num 696 ranges {5488640=458752,5947392=417792,6365184=753664,7118848=434176} 2024-09-13T14:26:04.978 INFO:tasks.rados.rados.0.smithi026.stdout:2153: writing smithi02683976-15 from 5488640 to 5947392 tid 1 2024-09-13T14:26:04.980 INFO:tasks.rados.rados.0.smithi026.stdout:2153: writing smithi02683976-15 from 5947392 to 6365184 tid 2 2024-09-13T14:26:04.984 INFO:tasks.rados.rados.0.smithi026.stdout:2153: writing smithi02683976-15 from 6365184 to 7118848 tid 3 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:2153: writing smithi02683976-15 from 7118848 to 7553024 tid 4 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:2154: snap_create 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:2151: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 0 (ObjNum 6145 snap 0 seq_num 3890193216) dirty dne 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 973 (ObjNum 685 snap 193 seq_num 685) dirty exists 2024-09-13T14:26:04.987 INFO:tasks.rados.rados.0.smithi026.stdout:2146: expect (ObjNum 690 snap 195 seq_num 690) 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1017 (ObjNum 678 snap 192 seq_num 678) dirty exists 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:2149: finishing rollback tid 0 to smithi02683976-24 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:2149: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:2149: finishing rollback tid 2 to smithi02683976-24 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 976 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:26:05.039 INFO:tasks.rados.rados.0.smithi026.stdout:2152: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 podman[95215]: 2024-09-13 14:26:04.902831047 +0000 UTC m=+0.348641162 container init 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:26:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 podman[95215]: 2024-09-13 14:26:04.907954611 +0000 UTC m=+0.353764720 container start 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:26:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:04 smithi145 ceph-osd[95251]: -- 172.21.15.145:0/2328658963 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55edeaffda40 con 0x55edebdca000 2024-09-13T14:26:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:05 smithi145 bash[95215]: 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 2024-09-13T14:26:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:05 smithi145 systemd[1]: Started Ceph osd.4 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:26:05.225 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 1 to smithi02683976-15 2024-09-13T14:26:05.225 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 2 to smithi02683976-15 2024-09-13T14:26:05.225 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 3 to smithi02683976-15 2024-09-13T14:26:05.226 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 4 to smithi02683976-15 2024-09-13T14:26:05.226 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 5 to smithi02683976-15 2024-09-13T14:26:05.226 INFO:tasks.rados.rados.0.smithi026.stdout:2153: finishing write tid 6 to smithi02683976-15 2024-09-13T14:26:05.226 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 677 (ObjNum 696 snap 196 seq_num 696) dirty exists 2024-09-13T14:26:05.226 INFO:tasks.rados.rados.0.smithi026.stdout:2153: left oid 15 (ObjNum 696 snap 196 seq_num 696) 2024-09-13T14:26:05.931 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:05.931 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:05.960 INFO:tasks.rados.rados.0.smithi026.stdout:2146: done (7 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2148: done (6 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2149: done (5 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2150: done (4 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2151: done (3 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2152: done (2 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2153: done (1 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2154: done (0 left) 2024-09-13T14:26:05.961 INFO:tasks.rados.rados.0.smithi026.stdout:2155: snap_create 2024-09-13T14:26:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-mon[89145]: pgmap v300: 121 pgs: 28 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 13 active+undersized+degraded, 72 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 16 op/s; 45/333 objects degraded (13.514%) 2024-09-13T14:26:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-mon[89145]: osdmap e468: 8 total, 7 up, 8 in 2024-09-13T14:26:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[97022]: pgmap v300: 121 pgs: 28 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 13 active+undersized+degraded, 72 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 16 op/s; 45/333 objects degraded (13.514%) 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[97022]: osdmap e468: 8 total, 7 up, 8 in 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[99461]: pgmap v300: 121 pgs: 28 active+undersized, 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 5 peering, 13 active+undersized+degraded, 72 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 16 op/s; 45/333 objects degraded (13.514%) 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[99461]: osdmap e468: 8 total, 7 up, 8 in 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:06.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:06.768 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:26:06.547+0000 7f1d84526640 -1 Falling back to public interface 2024-09-13T14:26:06.964 INFO:tasks.rados.rados.0.smithi026.stdout:2155: done (0 left) 2024-09-13T14:26:06.965 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 198 2024-09-13T14:26:06.965 INFO:tasks.rados.rados.0.smithi026.stdout:2156: seq_num 697 ranges {0=475136,475136=704512,1179648=557056,1736704=344064} 2024-09-13T14:26:06.967 INFO:tasks.rados.rados.0.smithi026.stdout:2156: writing smithi02683976-11 from 0 to 475136 tid 1 2024-09-13T14:26:06.971 INFO:tasks.rados.rados.0.smithi026.stdout:2156: writing smithi02683976-11 from 475136 to 1179648 tid 2 2024-09-13T14:26:06.974 INFO:tasks.rados.rados.0.smithi026.stdout:2156: writing smithi02683976-11 from 1179648 to 1736704 tid 3 2024-09-13T14:26:06.977 INFO:tasks.rados.rados.0.smithi026.stdout:2156: writing smithi02683976-11 from 1736704 to 2080768 tid 4 2024-09-13T14:26:06.977 INFO:tasks.rados.rados.0.smithi026.stdout:2157: delete oid 48 current snap is 198 2024-09-13T14:26:06.977 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 1 to smithi02683976-11 2024-09-13T14:26:06.977 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 2 to smithi02683976-11 2024-09-13T14:26:06.979 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 3 to smithi02683976-11 2024-09-13T14:26:06.980 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 4 to smithi02683976-11 2024-09-13T14:26:06.980 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 5 to smithi02683976-11 2024-09-13T14:26:06.981 INFO:tasks.rados.rados.0.smithi026.stdout:2156: finishing write tid 6 to smithi02683976-11 2024-09-13T14:26:06.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1093 (ObjNum 697 snap 198 seq_num 697) dirty exists 2024-09-13T14:26:06.981 INFO:tasks.rados.rados.0.smithi026.stdout:2156: left oid 11 (ObjNum 697 snap 198 seq_num 697) 2024-09-13T14:26:06.982 INFO:tasks.rados.rados.0.smithi026.stdout:2156: done (1 left) 2024-09-13T14:26:06.982 INFO:tasks.rados.rados.0.smithi026.stdout:2157: done (0 left) 2024-09-13T14:26:06.982 INFO:tasks.rados.rados.0.smithi026.stdout:2158: snap_create 2024-09-13T14:26:07.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:06 smithi145 ceph-mon[89145]: osdmap e469: 8 total, 7 up, 8 in 2024-09-13T14:26:07.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:06 smithi026 ceph-mon[97022]: osdmap e469: 8 total, 7 up, 8 in 2024-09-13T14:26:07.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:06 smithi026 ceph-mon[99461]: osdmap e469: 8 total, 7 up, 8 in 2024-09-13T14:26:07.965 INFO:tasks.rados.rados.0.smithi026.stdout:2158: done (0 left) 2024-09-13T14:26:07.966 INFO:tasks.rados.rados.0.smithi026.stdout:2159: read oid 1 snap -1 2024-09-13T14:26:07.966 INFO:tasks.rados.rados.0.smithi026.stdout:2159: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:26:07.966 INFO:tasks.rados.rados.0.smithi026.stdout:2160: read oid 17 snap -1 2024-09-13T14:26:07.966 INFO:tasks.rados.rados.0.smithi026.stdout:2160: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:26:07.966 INFO:tasks.rados.rados.0.smithi026.stdout:2161: snap_remove snap 197 2024-09-13T14:26:07.997 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:26:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:07 smithi145 ceph-mon[89145]: pgmap v303: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 52/309 objects degraded (16.828%) 2024-09-13T14:26:08.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:07 smithi145 ceph-mon[89145]: osdmap e470: 8 total, 7 up, 8 in 2024-09-13T14:26:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:07 smithi026 ceph-mon[97022]: pgmap v303: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 52/309 objects degraded (16.828%) 2024-09-13T14:26:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:07 smithi026 ceph-mon[97022]: osdmap e470: 8 total, 7 up, 8 in 2024-09-13T14:26:08.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:07 smithi026 ceph-mon[99461]: pgmap v303: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 52/309 objects degraded (16.828%) 2024-09-13T14:26:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:07 smithi026 ceph-mon[99461]: osdmap e470: 8 total, 7 up, 8 in 2024-09-13T14:26:08.589 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 32s ago 9m 18.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (4m) 2m ago 9m 80.5M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (10m) 32s ago 10m 15.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (10m) 2m ago 10m 21.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (3m) 2m ago 14m 452M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (4m) 32s ago 16m 523M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 32s ago 16m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (2m) 2m ago 14m 21.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 32s ago 14m 46.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:26:08.590 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 32s ago 9m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (4m) 2m ago 9m 8560k - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 32s ago 14m 152M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (103s) 32s ago 13m 144M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (70s) 32s ago 13m 108M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (36s) 32s ago 12m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 starting - - - 2480M 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (12m) 2m ago 12m 155M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 9910270fa529 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (11m) 2m ago 11m 107M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (11m) 2m ago 11m 120M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:26:08.591 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (3m) 2m ago 9m 41.9M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:26:08.936 INFO:tasks.rados.rados.0.smithi026.stdout:2161: done (2 left) 2024-09-13T14:26:08.936 INFO:tasks.rados.rados.0.smithi026.stdout:2162: rollback oid 11 current snap is 199 2024-09-13T14:26:08.936 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 190 2024-09-13T14:26:08.936 INFO:tasks.rados.rados.0.smithi026.stdout:2159: expect (ObjNum 636 snap 180 seq_num 636) 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:26:08.943 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 9 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:26:08.944 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:26:08.966 INFO:tasks.rados.rados.0.smithi026.stdout:2160: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:26:09.242 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "9/20 daemons upgraded", 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:26:09.243 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:26:09.243 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: osdmap e471: 8 total, 7 up, 8 in 2024-09-13T14:26:09.243 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.243 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 52/309 objects degraded (16.828%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: osdmap e472: 8 total, 7 up, 8 in 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1429805153' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: osdmap e471: 8 total, 7 up, 8 in 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 52/309 objects degraded (16.828%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: osdmap e472: 8 total, 7 up, 8 in 2024-09-13T14:26:09.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:08 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1429805153' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: osdmap e471: 8 total, 7 up, 8 in 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 52/309 objects degraded (16.828%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: osdmap e472: 8 total, 7 up, 8 in 2024-09-13T14:26:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:08 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1429805153' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:09.782 INFO:tasks.rados.rados.0.smithi026.stdout:2162: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:2162: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1096 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:2159: done (2 left) 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:2160: done (1 left) 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:2162: done (0 left) 2024-09-13T14:26:09.783 INFO:tasks.rados.rados.0.smithi026.stdout:2163: snap_create 2024-09-13T14:26:10.356 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:10.356 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:09 smithi145 ceph-mon[89145]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.356 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:09 smithi145 ceph-mon[89145]: pgmap v306: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 4.6 MiB/s wr, 13 op/s; 54/315 objects degraded (17.143%) 2024-09-13T14:26:10.357 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:09 smithi145 ceph-mon[89145]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.357 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:09 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.357 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:09 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[97022]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[97022]: pgmap v306: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 4.6 MiB/s wr, 13 op/s; 54/315 objects degraded (17.143%) 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[97022]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[99461]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[99461]: pgmap v306: 121 pgs: 32 active+undersized, 1 active+clean+snaptrim, 14 active+undersized+degraded, 74 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 4.6 MiB/s wr, 13 op/s; 54/315 objects degraded (17.143%) 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[99461]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:09 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:10.482 INFO:tasks.rados.rados.0.smithi026.stdout:2163: done (0 left) 2024-09-13T14:26:10.482 INFO:tasks.rados.rados.0.smithi026.stdout:2164: snap_remove snap 195 2024-09-13T14:26:10.674 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:26:10.355+0000 7f1d84526640 -1 osd.4 463 log_to_monitors true 2024-09-13T14:26:11.204 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:26:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:26:10.939+0000 7f1d782c8640 -1 osd.4 463 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:26:11.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.204 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:26:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: osdmap e473: 8 total, 7 up, 8 in 2024-09-13T14:26:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:11 smithi145 ceph-mon[89145]: from='osd.4 ' entity='osd.4' 2024-09-13T14:26:11.415 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:11] "GET /metrics HTTP/1.1" 200 36098 "" "Prometheus/2.51.0" 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: osdmap e473: 8 total, 7 up, 8 in 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[97022]: from='osd.4 ' entity='osd.4' 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: osdmap e473: 8 total, 7 up, 8 in 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:11 smithi026 ceph-mon[99461]: from='osd.4 ' entity='osd.4' 2024-09-13T14:26:11.490 INFO:tasks.rados.rados.0.smithi026.stdout:2164: done (0 left) 2024-09-13T14:26:11.490 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 200 2024-09-13T14:26:11.490 INFO:tasks.rados.rados.0.smithi026.stdout:2165: seq_num 699 ranges {0=516096,516096=475136,991232=573440,1564672=557056} 2024-09-13T14:26:11.493 INFO:tasks.rados.rados.0.smithi026.stdout:2165: writing smithi02683976-33 from 0 to 516096 tid 1 2024-09-13T14:26:11.497 INFO:tasks.rados.rados.0.smithi026.stdout:2165: writing smithi02683976-33 from 516096 to 991232 tid 2 2024-09-13T14:26:11.501 INFO:tasks.rados.rados.0.smithi026.stdout:2165: writing smithi02683976-33 from 991232 to 1564672 tid 3 2024-09-13T14:26:11.506 INFO:tasks.rados.rados.0.smithi026.stdout:2165: writing smithi02683976-33 from 1564672 to 2121728 tid 4 2024-09-13T14:26:11.506 INFO:tasks.rados.rados.0.smithi026.stdout:2166: snap_remove snap 190 2024-09-13T14:26:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: pgmap v309: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 212 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 54/318 objects degraded (16.981%) 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396] boot 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: pgmap v309: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 212 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 54/318 objects degraded (16.981%) 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:12.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396] boot 2024-09-13T14:26:12.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:26:12.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:26:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: pgmap v309: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 212 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 1.8 MiB/s wr, 4 op/s; 54/318 objects degraded (16.981%) 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: osd.4 [v2:172.21.15.145:6800/3668564396,v1:172.21.15.145:6801/3668564396] boot 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:26:12.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:26:12.493 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 1 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 2 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 3 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 4 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 5 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: finishing write tid 6 to smithi02683976-33 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1128 (ObjNum 699 snap 200 seq_num 699) dirty exists 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: left oid 33 (ObjNum 699 snap 200 seq_num 699) 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2165: done (1 left) 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2166: done (0 left) 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2167: read oid 10 snap -1 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2167: expect (ObjNum 694 snap 196 seq_num 694) 2024-09-13T14:26:12.494 INFO:tasks.rados.rados.0.smithi026.stdout:2168: snap_remove snap 198 2024-09-13T14:26:13.496 INFO:tasks.rados.rados.0.smithi026.stdout:2168: done (1 left) 2024-09-13T14:26:13.496 INFO:tasks.rados.rados.0.smithi026.stdout:2169: snap_remove snap 196 2024-09-13T14:26:13.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:26:13.371+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: pgmap v312: 121 pgs: 23 peering, 14 active+undersized, 9 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 508 KiB/s wr, 3 op/s; 31/321 objects degraded (9.657%) 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:13.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: pgmap v312: 121 pgs: 23 peering, 14 active+undersized, 9 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 508 KiB/s wr, 3 op/s; 31/321 objects degraded (9.657%) 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:13.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:13.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:13 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:26:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:26:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: pgmap v312: 121 pgs: 23 peering, 14 active+undersized, 9 active+undersized+degraded, 75 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 774 KiB/s rd, 508 KiB/s wr, 3 op/s; 31/321 objects degraded (9.657%) 2024-09-13T14:26:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:13.956 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:13 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T14:26:14.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:14.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:26:14.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:14.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:14.497 INFO:tasks.rados.rados.0.smithi026.stdout:2169: done (1 left) 2024-09-13T14:26:14.497 INFO:tasks.rados.rados.0.smithi026.stdout:2170: read oid 18 snap 199 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2170: expect deleted 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2171: read oid 28 snap -1 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2171: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2172: read oid 22 snap 199 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2172: expect (ObjNum 691 snap 195 seq_num 691) 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2173: read oid 9 snap -1 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2173: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2174: setattr oid 33 current snap is 200 2024-09-13T14:26:14.498 INFO:tasks.rados.rados.0.smithi026.stdout:2167: expect (ObjNum 694 snap 196 seq_num 694) 2024-09-13T14:26:14.578 INFO:tasks.rados.rados.0.smithi026.stdout:2171: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:14.659 INFO:tasks.rados.rados.0.smithi026.stdout:2173: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:14.844 INFO:tasks.rados.rados.0.smithi026.stdout:2172: expect (ObjNum 691 snap 195 seq_num 691) 2024-09-13T14:26:14.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:14 smithi026 ceph-mon[99461]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:26:14.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:14 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 31/321 objects degraded (9.657%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:14.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:14 smithi026 ceph-mon[97022]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:26:14.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:14 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 31/321 objects degraded (9.657%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:14.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:14 smithi145 ceph-mon[89145]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:26:14.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:14 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 31/321 objects degraded (9.657%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2167: done (5 left) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2170: done (4 left) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2171: done (3 left) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2172: done (2 left) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2173: done (1 left) 2024-09-13T14:26:15.112 INFO:tasks.rados.rados.0.smithi026.stdout:2175: snap_remove snap 199 2024-09-13T14:26:15.507 INFO:tasks.rados.rados.0.smithi026.stdout:2175: done (1 left) 2024-09-13T14:26:15.507 INFO:tasks.rados.rados.0.smithi026.stdout:2176: read oid 31 snap -1 2024-09-13T14:26:15.507 INFO:tasks.rados.rados.0.smithi026.stdout:2176: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:26:15.507 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 200 2024-09-13T14:26:15.508 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1135 (ObjNum 699 snap 200 seq_num 699) dirty exists 2024-09-13T14:26:15.508 INFO:tasks.rados.rados.0.smithi026.stdout:2177: seq_num 700 ranges {0=532480,532480=409600,942080=737280,1679360=458752} 2024-09-13T14:26:15.510 INFO:tasks.rados.rados.0.smithi026.stdout:2177: writing smithi02683976-2 from 0 to 532480 tid 1 2024-09-13T14:26:15.512 INFO:tasks.rados.rados.0.smithi026.stdout:2177: writing smithi02683976-2 from 532480 to 942080 tid 2 2024-09-13T14:26:15.517 INFO:tasks.rados.rados.0.smithi026.stdout:2177: writing smithi02683976-2 from 942080 to 1679360 tid 3 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2177: writing smithi02683976-2 from 1679360 to 2138112 tid 4 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2174: done (2 left) 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2178: read oid 36 snap -1 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2178: expect (ObjNum 690 snap 195 seq_num 690) 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2179: read oid 15 snap -1 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2179: expect (ObjNum 696 snap 196 seq_num 696) 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 200 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 1 to smithi02683976-2 2024-09-13T14:26:15.521 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 2 to smithi02683976-2 2024-09-13T14:26:15.522 INFO:tasks.rados.rados.0.smithi026.stdout:2180: seq_num 701 ranges {2121728=548864,2670592=696320,3366912=540672,3907584=368640} 2024-09-13T14:26:15.537 INFO:tasks.rados.rados.0.smithi026.stdout:2180: writing smithi02683976-33 from 2121728 to 2670592 tid 1 2024-09-13T14:26:15.541 INFO:tasks.rados.rados.0.smithi026.stdout:2180: writing smithi02683976-33 from 2670592 to 3366912 tid 2 2024-09-13T14:26:15.544 INFO:tasks.rados.rados.0.smithi026.stdout:2180: writing smithi02683976-33 from 3366912 to 3907584 tid 3 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2180: writing smithi02683976-33 from 3907584 to 4276224 tid 4 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2181: snap_create 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 3 to smithi02683976-2 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 4 to smithi02683976-2 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 5 to smithi02683976-2 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2177: finishing write tid 6 to smithi02683976-2 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 981 (ObjNum 700 snap 200 seq_num 700) dirty exists 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2177: left oid 2 (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:26:15.547 INFO:tasks.rados.rados.0.smithi026.stdout:2178: expect (ObjNum 690 snap 195 seq_num 690) 2024-09-13T14:26:15.601 INFO:tasks.rados.rados.0.smithi026.stdout:2176: expect (ObjNum 614 snap 176 seq_num 614) 2024-09-13T14:26:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:15 smithi026 ceph-mon[99461]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:26:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:15 smithi026 ceph-mon[99461]: pgmap v315: 121 pgs: 23 peering, 10 active+undersized, 8 active+undersized+degraded, 80 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 508 KiB/s wr, 1 op/s; 29/318 objects degraded (9.119%) 2024-09-13T14:26:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:15 smithi026 ceph-mon[97022]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:26:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:15 smithi026 ceph-mon[97022]: pgmap v315: 121 pgs: 23 peering, 10 active+undersized, 8 active+undersized+degraded, 80 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 508 KiB/s wr, 1 op/s; 29/318 objects degraded (9.119%) 2024-09-13T14:26:15.932 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 1 to smithi02683976-33 2024-09-13T14:26:15.932 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 2 to smithi02683976-33 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 3 to smithi02683976-33 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 4 to smithi02683976-33 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 5 to smithi02683976-33 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2180: finishing write tid 6 to smithi02683976-33 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1140 (ObjNum 701 snap 200 seq_num 701) dirty exists 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2180: left oid 33 (ObjNum 701 snap 200 seq_num 701) 2024-09-13T14:26:15.933 INFO:tasks.rados.rados.0.smithi026.stdout:2179: expect (ObjNum 696 snap 196 seq_num 696) 2024-09-13T14:26:15.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:15 smithi145 ceph-mon[89145]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:26:15.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:15 smithi145 ceph-mon[89145]: pgmap v315: 121 pgs: 23 peering, 10 active+undersized, 8 active+undersized+degraded, 80 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 508 KiB/s wr, 1 op/s; 29/318 objects degraded (9.119%) 2024-09-13T14:26:15.955 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:15.956 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2176: done (5 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2177: done (4 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2178: done (3 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2179: done (2 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2180: done (1 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2181: done (0 left) 2024-09-13T14:26:16.511 INFO:tasks.rados.rados.0.smithi026.stdout:2182: snap_remove snap 200 2024-09-13T14:26:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:16 smithi026 ceph-mon[97022]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:26:16.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:16 smithi026 ceph-mon[99461]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:26:16.949 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:16 smithi145 ceph-mon[89145]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:26:17.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:17.518 INFO:tasks.rados.rados.0.smithi026.stdout:2182: done (0 left) 2024-09-13T14:26:17.519 INFO:tasks.rados.rados.0.smithi026.stdout:2183: rollback oid 34 current snap is 201 2024-09-13T14:26:17.519 INFO:tasks.rados.rados.0.smithi026.stdout:2183: done (0 left) 2024-09-13T14:26:17.519 INFO:tasks.rados.rados.0.smithi026.stdout:2184: rmattr oid 49 current snap is 201 2024-09-13T14:26:17.519 INFO:tasks.rados.rados.0.smithi026.stdout:2185: copy_from oid 27 from oid 36 current snap is 201 2024-09-13T14:26:17.519 INFO:tasks.rados.rados.0.smithi026.stdout:2186: snap_create 2024-09-13T14:26:17.524 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 946 (ObjNum 676 snap 191 seq_num 676) dirty exists 2024-09-13T14:26:17.529 INFO:tasks.rados.rados.0.smithi026.stdout:2185: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:26:17.530 INFO:tasks.rados.rados.0.smithi026.stdout:2185: finishing copy_from to smithi02683976-27 2024-09-13T14:26:17.530 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1099 (ObjNum 690 snap 195 seq_num 690) dirty exists 2024-09-13T14:26:17.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:17 smithi026 ceph-mon[99461]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:26:17.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:17 smithi026 ceph-mon[99461]: pgmap v318: 121 pgs: 4 peering, 117 active+clean; 194 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 518 KiB/s wr, 8 op/s; 4.3 MiB/s, 1 objects/s recovering 2024-09-13T14:26:17.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:17 smithi026 ceph-mon[97022]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:26:17.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:17 smithi026 ceph-mon[97022]: pgmap v318: 121 pgs: 4 peering, 117 active+clean; 194 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 518 KiB/s wr, 8 op/s; 4.3 MiB/s, 1 objects/s recovering 2024-09-13T14:26:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:17 smithi145 ceph-mon[89145]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:26:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:17 smithi145 ceph-mon[89145]: pgmap v318: 121 pgs: 4 peering, 117 active+clean; 194 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 518 KiB/s wr, 8 op/s; 4.3 MiB/s, 1 objects/s recovering 2024-09-13T14:26:18.522 INFO:tasks.rados.rados.0.smithi026.stdout:2184: done (2 left) 2024-09-13T14:26:18.523 INFO:tasks.rados.rados.0.smithi026.stdout:2185: done (1 left) 2024-09-13T14:26:18.523 INFO:tasks.rados.rados.0.smithi026.stdout:2186: done (0 left) 2024-09-13T14:26:18.523 INFO:tasks.rados.rados.0.smithi026.stdout:2187: delete oid 6 current snap is 202 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2187: done (0 left) 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2188: read oid 47 snap -1 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2188: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2189: read oid 2 snap -1 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2189: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2190: read oid 42 snap -1 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2190: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2191: copy_from oid 7 from oid 24 current snap is 202 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2192: copy_from oid 1 from oid 14 current snap is 202 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 202 2024-09-13T14:26:18.529 INFO:tasks.rados.rados.0.smithi026.stdout:2193: seq_num 703 ranges {4276224=581632,4857856=458752,5316608=557056,5873664=417792,6291456=172032} 2024-09-13T14:26:18.557 INFO:tasks.rados.rados.0.smithi026.stdout:2193: writing smithi02683976-33 from 4276224 to 4857856 tid 1 2024-09-13T14:26:18.559 INFO:tasks.rados.rados.0.smithi026.stdout:2193: writing smithi02683976-33 from 4857856 to 5316608 tid 2 2024-09-13T14:26:18.563 INFO:tasks.rados.rados.0.smithi026.stdout:2193: writing smithi02683976-33 from 5316608 to 5873664 tid 3 2024-09-13T14:26:18.565 INFO:tasks.rados.rados.0.smithi026.stdout:2193: writing smithi02683976-33 from 5873664 to 6291456 tid 4 2024-09-13T14:26:18.567 INFO:tasks.rados.rados.0.smithi026.stdout:2193: writing smithi02683976-33 from 6291456 to 6463488 tid 5 2024-09-13T14:26:18.567 INFO:tasks.rados.rados.0.smithi026.stdout:2194: read oid 41 snap -1 2024-09-13T14:26:18.567 INFO:tasks.rados.rados.0.smithi026.stdout:2194: expect (ObjNum 683 snap 193 seq_num 683) 2024-09-13T14:26:18.567 INFO:tasks.rados.rados.0.smithi026.stdout:2195: rollback oid 17 current snap is 202 2024-09-13T14:26:18.568 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 201 2024-09-13T14:26:18.568 INFO:tasks.rados.rados.0.smithi026.stdout:2190: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:18.651 INFO:tasks.rados.rados.0.smithi026.stdout:2189: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:26:18.707 INFO:tasks.rados.rados.0.smithi026.stdout:2188: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:18.886 INFO:tasks.rados.rados.0.smithi026.stdout:2191: finishing copy_from racing read to smithi02683976-7 2024-09-13T14:26:19.162 INFO:tasks.rados.rados.0.smithi026.stdout:2191: finishing copy_from to smithi02683976-7 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 976 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2192: finishing copy_from racing read to smithi02683976-1 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2188: done (7 left) 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2189: done (6 left) 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2190: done (5 left) 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2191: done (4 left) 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2196: delete oid 15 current snap is 202 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 1 to smithi02683976-33 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2192: finishing copy_from to smithi02683976-1 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 844 (ObjNum 616 snap 176 seq_num 616) dirty exists 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 2 to smithi02683976-33 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 3 to smithi02683976-33 2024-09-13T14:26:19.163 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 4 to smithi02683976-33 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 5 to smithi02683976-33 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 6 to smithi02683976-33 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:2193: finishing write tid 7 to smithi02683976-33 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1150 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:2193: left oid 33 (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:26:19.164 INFO:tasks.rados.rados.0.smithi026.stdout:2194: expect (ObjNum 683 snap 193 seq_num 683) 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:18 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/318 objects degraded (9.119%), 8 pgs degraded) 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:18 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:18 smithi145 ceph-mon[89145]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/318 objects degraded (9.119%), 8 pgs degraded) 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[99461]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:26:19.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/318 objects degraded (9.119%), 8 pgs degraded) 2024-09-13T14:26:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:26:19.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:18 smithi026 ceph-mon[97022]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2195: finishing rollback tid 0 to smithi02683976-17 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2195: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2195: finishing rollback tid 2 to smithi02683976-17 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 828 (ObjNum 578 snap 164 seq_num 578) dirty exists 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2192: done (4 left) 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2193: done (3 left) 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2194: done (2 left) 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2195: done (1 left) 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2196: done (0 left) 2024-09-13T14:26:19.887 INFO:tasks.rados.rados.0.smithi026.stdout:2197: snap_create 2024-09-13T14:26:19.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:19 smithi026 ceph-mon[97022]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:26:19.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:19 smithi026 ceph-mon[97022]: pgmap v321: 121 pgs: 121 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 1.0 MiB/s wr, 9 op/s; 7.2 MiB/s, 2 objects/s recovering 2024-09-13T14:26:19.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:19 smithi026 ceph-mon[99461]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:26:19.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:19 smithi026 ceph-mon[99461]: pgmap v321: 121 pgs: 121 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 1.0 MiB/s wr, 9 op/s; 7.2 MiB/s, 2 objects/s recovering 2024-09-13T14:26:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:19 smithi145 ceph-mon[89145]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:26:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:19 smithi145 ceph-mon[89145]: pgmap v321: 121 pgs: 121 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 1.0 MiB/s wr, 9 op/s; 7.2 MiB/s, 2 objects/s recovering 2024-09-13T14:26:20.528 INFO:tasks.rados.rados.0.smithi026.stdout:2197: done (0 left) 2024-09-13T14:26:20.528 INFO:tasks.rados.rados.0.smithi026.stdout:2198: snap_create 2024-09-13T14:26:20.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2198: done (0 left) 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2199: rmattr oid 40 current snap is 204 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2199: done (0 left) 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2200: rmattr oid 32 current snap is 204 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2200: done (0 left) 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2201: rollback oid 31 current snap is 204 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 202 2024-09-13T14:26:21.534 INFO:tasks.rados.rados.0.smithi026.stdout:2202: snap_create 2024-09-13T14:26:21.541 INFO:tasks.rados.rados.0.smithi026.stdout:2201: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:26:21.541 INFO:tasks.rados.rados.0.smithi026.stdout:2201: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:26:21.541 INFO:tasks.rados.rados.0.smithi026.stdout:2201: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:26:21.541 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1106 (ObjNum 614 snap 176 seq_num 614) dirty exists 2024-09-13T14:26:21.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:21 smithi026 ceph-mon[97022]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:26:21.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:21 smithi026 ceph-mon[97022]: pgmap v323: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 19 op/s; 4.9 MiB/s, 1 objects/s recovering 2024-09-13T14:26:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:21 smithi026 ceph-mon[99461]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:26:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:21 smithi026 ceph-mon[99461]: pgmap v323: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 19 op/s; 4.9 MiB/s, 1 objects/s recovering 2024-09-13T14:26:21.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:21] "GET /metrics HTTP/1.1" 200 36098 "" "Prometheus/2.51.0" 2024-09-13T14:26:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:21 smithi145 ceph-mon[89145]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:26:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:21 smithi145 ceph-mon[89145]: pgmap v323: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 19 op/s; 4.9 MiB/s, 1 objects/s recovering 2024-09-13T14:26:22.536 INFO:tasks.rados.rados.0.smithi026.stdout:2201: done (1 left) 2024-09-13T14:26:22.536 INFO:tasks.rados.rados.0.smithi026.stdout:2202: done (0 left) 2024-09-13T14:26:22.536 INFO:tasks.rados.rados.0.smithi026.stdout:2203: rollback oid 31 current snap is 205 2024-09-13T14:26:22.536 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 202 2024-09-13T14:26:22.536 INFO:tasks.rados.rados.0.smithi026.stdout:2204: snap_remove snap 201 2024-09-13T14:26:22.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:22 smithi026 ceph-mon[97022]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:26:22.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:22 smithi026 ceph-mon[99461]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:26:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:22 smithi145 ceph-mon[89145]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:26:23.539 INFO:tasks.rados.rados.0.smithi026.stdout:2204: done (1 left) 2024-09-13T14:26:23.539 INFO:tasks.rados.rados.0.smithi026.stdout:2205: rmattr oid 21 current snap is 205 2024-09-13T14:26:23.539 INFO:tasks.rados.rados.0.smithi026.stdout:2206: snap_remove snap 203 2024-09-13T14:26:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[97022]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:26:23.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[97022]: pgmap v326: 121 pgs: 121 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-13T14:26:23.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[97022]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:26:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[99461]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:26:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[99461]: pgmap v326: 121 pgs: 121 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-13T14:26:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:23 smithi026 ceph-mon[99461]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2206: done (2 left) 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2207: rollback oid 44 current snap is 205 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 202 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2208: read oid 16 snap -1 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2208: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2203: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2203: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:26:23.943 INFO:tasks.rados.rados.0.smithi026.stdout:2203: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:26:23.944 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1110 (ObjNum 614 snap 176 seq_num 614) dirty exists 2024-09-13T14:26:23.944 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1112 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:26:23.944 INFO:tasks.rados.rados.0.smithi026.stdout:2203: done (3 left) 2024-09-13T14:26:23.944 INFO:tasks.rados.rados.0.smithi026.stdout:2205: done (2 left) 2024-09-13T14:26:23.944 INFO:tasks.rados.rados.0.smithi026.stdout:2209: snap_create 2024-09-13T14:26:23.946 INFO:tasks.rados.rados.0.smithi026.stdout:2207: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:26:23.946 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 0 (ObjNum 177 snap 0 seq_num 519836624) dirty dne 2024-09-13T14:26:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:23 smithi145 ceph-mon[89145]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:26:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:23 smithi145 ceph-mon[89145]: pgmap v326: 121 pgs: 121 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.2 MiB/s wr, 16 op/s 2024-09-13T14:26:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:23 smithi145 ceph-mon[89145]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:26:23.971 INFO:tasks.rados.rados.0.smithi026.stdout:2208: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:26:24.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:24.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:26:24.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:2207: done (2 left) 2024-09-13T14:26:24.947 INFO:tasks.rados.rados.0.smithi026.stdout:2208: done (1 left) 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2209: done (0 left) 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2210: read oid 30 snap -1 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2210: expect (ObjNum 669 snap 188 seq_num 669) 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2211: read oid 46 snap 202 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2211: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2212: rmattr oid 39 current snap is 206 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2212: done (2 left) 2024-09-13T14:26:24.948 INFO:tasks.rados.rados.0.smithi026.stdout:2213: snap_create 2024-09-13T14:26:24.974 INFO:tasks.rados.rados.0.smithi026.stdout:2210: expect (ObjNum 669 snap 188 seq_num 669) 2024-09-13T14:26:25.083 INFO:tasks.rados.rados.0.smithi026.stdout:2211: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:26:25.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:24 smithi145 ceph-mon[89145]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:26:25.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:24 smithi026 ceph-mon[97022]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:26:25.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:24 smithi026 ceph-mon[99461]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:26:25.949 INFO:tasks.rados.rados.0.smithi026.stdout:2210: done (2 left) 2024-09-13T14:26:25.949 INFO:tasks.rados.rados.0.smithi026.stdout:2211: done (1 left) 2024-09-13T14:26:25.949 INFO:tasks.rados.rados.0.smithi026.stdout:2213: done (0 left) 2024-09-13T14:26:25.949 INFO:tasks.rados.rados.0.smithi026.stdout:2214: read oid 32 snap 206 2024-09-13T14:26:25.950 INFO:tasks.rados.rados.0.smithi026.stdout:2214: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:25.950 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 207 2024-09-13T14:26:25.950 INFO:tasks.rados.rados.0.smithi026.stdout:2215: seq_num 705 ranges {4497408=614400,5111808=630784,5742592=524288,6266880=450560} 2024-09-13T14:26:25.977 INFO:tasks.rados.rados.0.smithi026.stdout:2215: writing smithi02683976-46 from 4497408 to 5111808 tid 1 2024-09-13T14:26:25.982 INFO:tasks.rados.rados.0.smithi026.stdout:2215: writing smithi02683976-46 from 5111808 to 5742592 tid 2 2024-09-13T14:26:25.984 INFO:tasks.rados.rados.0.smithi026.stdout:2215: writing smithi02683976-46 from 5742592 to 6266880 tid 3 2024-09-13T14:26:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:2215: writing smithi02683976-46 from 6266880 to 6717440 tid 4 2024-09-13T14:26:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 207 2024-09-13T14:26:25.987 INFO:tasks.rados.rados.0.smithi026.stdout:2216: seq_num 706 ranges {8454144=630784,9084928=516096,9601024=729088,10330112=360448} 2024-09-13T14:26:26.036 INFO:tasks.rados.rados.0.smithi026.stdout:2216: writing smithi02683976-5 from 8454144 to 9084928 tid 1 2024-09-13T14:26:26.040 INFO:tasks.rados.rados.0.smithi026.stdout:2216: writing smithi02683976-5 from 9084928 to 9601024 tid 2 2024-09-13T14:26:26.044 INFO:tasks.rados.rados.0.smithi026.stdout:2216: writing smithi02683976-5 from 9601024 to 10330112 tid 3 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2216: writing smithi02683976-5 from 10330112 to 10690560 tid 4 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2217: read oid 50 snap -1 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2217: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2218: read oid 26 snap -1 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2218: expect (ObjNum 605 snap 172 seq_num 605) 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2219: copy_from oid 37 from oid 23 current snap is 207 2024-09-13T14:26:26.047 INFO:tasks.rados.rados.0.smithi026.stdout:2214: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 1 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 2 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 3 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 4 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 5 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: finishing write tid 6 to smithi02683976-46 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 926 (ObjNum 705 snap 207 seq_num 705) dirty exists 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2215: left oid 46 (ObjNum 705 snap 207 seq_num 705) 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 1 to smithi02683976-5 2024-09-13T14:26:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 2 to smithi02683976-5 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 3 to smithi02683976-5 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:2214: done (5 left) 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:2215: done (4 left) 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 207 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 4 to smithi02683976-5 2024-09-13T14:26:26.138 INFO:tasks.rados.rados.0.smithi026.stdout:2220: seq_num 707 ranges {13787136=647168,14434304=802816,15237120=532480,15769600=270336} 2024-09-13T14:26:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:25 smithi145 ceph-mon[89145]: pgmap v329: 121 pgs: 121 active+clean; 201 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-13T14:26:26.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:25 smithi145 ceph-mon[89145]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:26:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:26.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:26.215 INFO:tasks.rados.rados.0.smithi026.stdout:2220: writing smithi02683976-45 from 13787136 to 14434304 tid 1 2024-09-13T14:26:26.219 INFO:tasks.rados.rados.0.smithi026.stdout:2220: writing smithi02683976-45 from 14434304 to 15237120 tid 2 2024-09-13T14:26:26.223 INFO:tasks.rados.rados.0.smithi026.stdout:2220: writing smithi02683976-45 from 15237120 to 15769600 tid 3 2024-09-13T14:26:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:2220: writing smithi02683976-45 from 15769600 to 16039936 tid 4 2024-09-13T14:26:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:2221: rollback oid 25 current snap is 207 2024-09-13T14:26:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 205 2024-09-13T14:26:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:2218: expect (ObjNum 605 snap 172 seq_num 605) 2024-09-13T14:26:26.242 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 5 to smithi02683976-5 2024-09-13T14:26:26.242 INFO:tasks.rados.rados.0.smithi026.stdout:2216: finishing write tid 6 to smithi02683976-5 2024-09-13T14:26:26.242 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1160 (ObjNum 706 snap 207 seq_num 706) dirty exists 2024-09-13T14:26:26.242 INFO:tasks.rados.rados.0.smithi026.stdout:2216: left oid 5 (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:26:26.242 INFO:tasks.rados.rados.0.smithi026.stdout:2217: expect (ObjNum 651 snap 183 seq_num 651) 2024-09-13T14:26:26.277 INFO:tasks.rados.rados.0.smithi026.stdout:2219: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:26:26.277 INFO:tasks.rados.rados.0.smithi026.stdout:2219: finishing copy_from to smithi02683976-37 2024-09-13T14:26:26.277 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1017 (ObjNum 678 snap 192 seq_num 678) dirty exists 2024-09-13T14:26:26.277 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 1 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 2 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 3 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 4 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 5 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2221: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2221: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2221: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 984 (ObjNum 685 snap 193 seq_num 685) dirty exists 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: finishing write tid 6 to smithi02683976-45 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 832 (ObjNum 707 snap 207 seq_num 707) dirty exists 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2220: left oid 45 (ObjNum 707 snap 207 seq_num 707) 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2216: done (5 left) 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2217: done (4 left) 2024-09-13T14:26:26.278 INFO:tasks.rados.rados.0.smithi026.stdout:2218: done (3 left) 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2219: done (2 left) 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2220: done (1 left) 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2221: done (0 left) 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2222: read oid 32 snap -1 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2222: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2223: rollback oid 37 current snap is 207 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 205 2024-09-13T14:26:26.279 INFO:tasks.rados.rados.0.smithi026.stdout:2224: delete oid 45 current snap is 207 2024-09-13T14:26:26.281 INFO:tasks.rados.rados.0.smithi026.stdout:2223: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:26:26.281 INFO:tasks.rados.rados.0.smithi026.stdout:2224: done (2 left) 2024-09-13T14:26:26.281 INFO:tasks.rados.rados.0.smithi026.stdout:2225: read oid 43 snap -1 2024-09-13T14:26:26.285 INFO:tasks.rados.rados.0.smithi026.stdout:2225: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:26:26.285 INFO:tasks.rados.rados.0.smithi026.stdout:2226: read oid 9 snap -1 2024-09-13T14:26:26.285 INFO:tasks.rados.rados.0.smithi026.stdout:2226: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:26.285 INFO:tasks.rados.rados.0.smithi026.stdout:2227: snap_create 2024-09-13T14:26:26.285 INFO:tasks.rados.rados.0.smithi026.stdout:2223: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:26:26.286 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1019 (ObjNum 833 snap 0 seq_num 518839520) dirty dne 2024-09-13T14:26:26.286 INFO:tasks.rados.rados.0.smithi026.stdout:2222: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:26.372 INFO:tasks.rados.rados.0.smithi026.stdout:2225: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:26:26.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:25 smithi026 ceph-mon[99461]: pgmap v329: 121 pgs: 121 active+clean; 201 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-13T14:26:26.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:25 smithi026 ceph-mon[99461]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:26:26.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:25 smithi026 ceph-mon[97022]: pgmap v329: 121 pgs: 121 active+clean; 201 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 2 op/s 2024-09-13T14:26:26.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:25 smithi026 ceph-mon[97022]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:26:26.437 INFO:tasks.rados.rados.0.smithi026.stdout:2226: expect (ObjNum 638 snap 180 seq_num 638) 2024-09-13T14:26:27.205 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:27.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:26 smithi145 ceph-mon[89145]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:26:27.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:27.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2222: done (4 left) 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2223: done (3 left) 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2225: done (2 left) 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2226: done (1 left) 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2227: done (0 left) 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2228: copy_from oid 45 from oid 2 current snap is 208 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2229: read oid 19 snap 202 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2229: expect deleted 2024-09-13T14:26:27.221 INFO:tasks.rados.rados.0.smithi026.stdout:2230: setattr oid 40 current snap is 208 2024-09-13T14:26:27.222 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 208 2024-09-13T14:26:27.222 INFO:tasks.rados.rados.0.smithi026.stdout:2231: seq_num 709 ranges {5505024=679936,6184960=614400,6799360=499712,7299072=491520} 2024-09-13T14:26:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:2231: writing smithi02683976-49 from 5505024 to 6184960 tid 1 2024-09-13T14:26:27.259 INFO:tasks.rados.rados.0.smithi026.stdout:2231: writing smithi02683976-49 from 6184960 to 6799360 tid 2 2024-09-13T14:26:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:2231: writing smithi02683976-49 from 6799360 to 7299072 tid 3 2024-09-13T14:26:27.266 INFO:tasks.rados.rados.0.smithi026.stdout:2231: writing smithi02683976-49 from 7299072 to 7790592 tid 4 2024-09-13T14:26:27.266 INFO:tasks.rados.rados.0.smithi026.stdout:2232: rmattr oid 27 current snap is 208 2024-09-13T14:26:27.266 INFO:tasks.rados.rados.0.smithi026.stdout:2232: done (4 left) 2024-09-13T14:26:27.266 INFO:tasks.rados.rados.0.smithi026.stdout:2233: rmattr oid 15 current snap is 208 2024-09-13T14:26:27.266 INFO:tasks.rados.rados.0.smithi026.stdout:2233: done (4 left) 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2234: delete oid 44 current snap is 208 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 947 (ObjNum 65 snap 0 seq_num 1364748371) dirty exists 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2228: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2228: finishing copy_from to smithi02683976-45 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 981 (ObjNum 700 snap 200 seq_num 700) dirty exists 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 1 to smithi02683976-49 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 2 to smithi02683976-49 2024-09-13T14:26:27.267 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 3 to smithi02683976-49 2024-09-13T14:26:27.268 INFO:tasks.rados.rados.0.smithi026.stdout:2228: done (4 left) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2229: done (3 left) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2230: done (2 left) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2234: done (1 left) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2235: rmattr oid 48 current snap is 208 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2235: done (1 left) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2236: read oid 38 snap 204 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2236: expect (ObjNum 673 snap 189 seq_num 673) 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2237: rollback oid 2 current snap is 208 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 204 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2238: setattr oid 43 current snap is 208 2024-09-13T14:26:27.269 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 4 to smithi02683976-49 2024-09-13T14:26:27.270 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 5 to smithi02683976-49 2024-09-13T14:26:27.271 INFO:tasks.rados.rados.0.smithi026.stdout:2231: finishing write tid 6 to smithi02683976-49 2024-09-13T14:26:27.271 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 952 (ObjNum 709 snap 208 seq_num 709) dirty exists 2024-09-13T14:26:27.271 INFO:tasks.rados.rados.0.smithi026.stdout:2231: left oid 49 (ObjNum 709 snap 208 seq_num 709) 2024-09-13T14:26:27.272 INFO:tasks.rados.rados.0.smithi026.stdout:2231: done (3 left) 2024-09-13T14:26:27.272 INFO:tasks.rados.rados.0.smithi026.stdout:2239: read oid 44 snap -1 2024-09-13T14:26:27.272 INFO:tasks.rados.rados.0.smithi026.stdout:2239: expect deleted 2024-09-13T14:26:27.272 INFO:tasks.rados.rados.0.smithi026.stdout:2240: snap_remove snap 206 2024-09-13T14:26:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[97022]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:26:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[99461]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:26:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:28.227 INFO:tasks.rados.rados.0.smithi026.stdout:2240: done (4 left) 2024-09-13T14:26:28.227 INFO:tasks.rados.rados.0.smithi026.stdout:2241: copy_from oid 40 from oid 47 current snap is 208 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:2237: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:2237: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:2237: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 987 (ObjNum 700 snap 200 seq_num 700) dirty exists 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1065 (ObjNum 660 snap 188 seq_num 660) dirty exists 2024-09-13T14:26:28.228 INFO:tasks.rados.rados.0.smithi026.stdout:2236: expect (ObjNum 673 snap 189 seq_num 673) 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:2241: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:2241: finishing copy_from to smithi02683976-40 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 758 (ObjNum 693 snap 195 seq_num 693) dirty exists 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:2236: done (4 left) 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:2237: done (3 left) 2024-09-13T14:26:28.277 INFO:tasks.rados.rados.0.smithi026.stdout:2238: done (2 left) 2024-09-13T14:26:28.278 INFO:tasks.rados.rados.0.smithi026.stdout:2239: done (1 left) 2024-09-13T14:26:28.278 INFO:tasks.rados.rados.0.smithi026.stdout:2241: done (0 left) 2024-09-13T14:26:28.278 INFO:tasks.rados.rados.0.smithi026.stdout:2242: delete oid 4 current snap is 208 2024-09-13T14:26:28.279 INFO:tasks.rados.rados.0.smithi026.stdout:2242: done (0 left) 2024-09-13T14:26:28.279 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 208 2024-09-13T14:26:28.279 INFO:tasks.rados.rados.0.smithi026.stdout:2243: seq_num 712 ranges {8675328=729088,9404416=671744,10076160=671744,10747904=262144} 2024-09-13T14:26:28.330 INFO:tasks.rados.rados.0.smithi026.stdout:2243: writing smithi02683976-23 from 8675328 to 9404416 tid 1 2024-09-13T14:26:28.334 INFO:tasks.rados.rados.0.smithi026.stdout:2243: writing smithi02683976-23 from 9404416 to 10076160 tid 2 2024-09-13T14:26:28.338 INFO:tasks.rados.rados.0.smithi026.stdout:2243: writing smithi02683976-23 from 10076160 to 10747904 tid 3 2024-09-13T14:26:28.340 INFO:tasks.rados.rados.0.smithi026.stdout:2243: writing smithi02683976-23 from 10747904 to 11010048 tid 4 2024-09-13T14:26:28.340 INFO:tasks.rados.rados.0.smithi026.stdout:2244: snap_remove snap 207 2024-09-13T14:26:28.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:28 smithi026 ceph-mon[99461]: pgmap v332: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:26:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:28 smithi026 ceph-mon[99461]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:26:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:28 smithi026 ceph-mon[97022]: pgmap v332: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:26:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:28 smithi026 ceph-mon[97022]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:26:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:28 smithi145 ceph-mon[89145]: pgmap v332: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:26:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:28 smithi145 ceph-mon[89145]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:26:29.235 INFO:tasks.rados.rados.0.smithi026.stdout:2244: done (1 left) 2024-09-13T14:26:29.235 INFO:tasks.rados.rados.0.smithi026.stdout:2245: delete oid 30 current snap is 208 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 1 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 2 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 3 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 4 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 5 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: finishing write tid 6 to smithi02683976-23 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1027 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:26:29.236 INFO:tasks.rados.rados.0.smithi026.stdout:2243: left oid 23 (ObjNum 712 snap 208 seq_num 712) 2024-09-13T14:26:29.237 INFO:tasks.rados.rados.0.smithi026.stdout:2243: done (1 left) 2024-09-13T14:26:29.237 INFO:tasks.rados.rados.0.smithi026.stdout:2245: done (0 left) 2024-09-13T14:26:29.237 INFO:tasks.rados.rados.0.smithi026.stdout:2246: setattr oid 22 current snap is 208 2024-09-13T14:26:29.238 INFO:tasks.rados.rados.0.smithi026.stdout:2247: snap_create 2024-09-13T14:26:29.241 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 969 (ObjNum 691 snap 195 seq_num 691) dirty exists 2024-09-13T14:26:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:29 smithi145 ceph-mon[89145]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:26:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:29 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:29 smithi145 ceph-mon[89145]: Upgrade: osd.5 is safe to restart 2024-09-13T14:26:29.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[99461]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:26:29.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[99461]: Upgrade: osd.5 is safe to restart 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[97022]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T14:26:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:29 smithi026 ceph-mon[97022]: Upgrade: osd.5 is safe to restart 2024-09-13T14:26:30.236 INFO:tasks.rados.rados.0.smithi026.stdout:2246: done (1 left) 2024-09-13T14:26:30.236 INFO:tasks.rados.rados.0.smithi026.stdout:2247: done (0 left) 2024-09-13T14:26:30.236 INFO:tasks.rados.rados.0.smithi026.stdout:2248: snap_remove snap 205 2024-09-13T14:26:30.445 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-mon[89145]: pgmap v335: 121 pgs: 121 active+clean; 179 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 542 KiB/s wr, 9 op/s 2024-09-13T14:26:30.445 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-mon[89145]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:26:30.446 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:30.446 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:26:30.446 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:30.446 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[97022]: pgmap v335: 121 pgs: 121 active+clean; 179 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 542 KiB/s wr, 9 op/s 2024-09-13T14:26:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[97022]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[99461]: pgmap v335: 121 pgs: 121 active+clean; 179 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 542 KiB/s wr, 9 op/s 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[99461]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:26:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:30 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:31.238 INFO:tasks.rados.rados.0.smithi026.stdout:2248: done (0 left) 2024-09-13T14:26:31.238 INFO:tasks.rados.rados.0.smithi026.stdout:2249: read oid 8 snap -1 2024-09-13T14:26:31.238 INFO:tasks.rados.rados.0.smithi026.stdout:2249: expect (ObjNum 679 snap 192 seq_num 679) 2024-09-13T14:26:31.238 INFO:tasks.rados.rados.0.smithi026.stdout:2250: read oid 35 snap -1 2024-09-13T14:26:31.238 INFO:tasks.rados.rados.0.smithi026.stdout:2250: expect (ObjNum 687 snap 193 seq_num 687) 2024-09-13T14:26:31.239 INFO:tasks.rados.rados.0.smithi026.stdout:2251: rollback oid 13 current snap is 209 2024-09-13T14:26:31.239 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 202 2024-09-13T14:26:31.239 INFO:tasks.rados.rados.0.smithi026.stdout:2252: snap_remove snap 208 2024-09-13T14:26:31.519 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:31 smithi145 ceph-mon[89145]: Upgrade: Updating osd.5 2024-09-13T14:26:31.519 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:31 smithi145 ceph-mon[89145]: Deploying daemon osd.5 on smithi145 2024-09-13T14:26:31.520 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:31 smithi145 ceph-mon[89145]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:26:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:31] "GET /metrics HTTP/1.1" 200 36239 "" "Prometheus/2.51.0" 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[97022]: Upgrade: Updating osd.5 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[97022]: Deploying daemon osd.5 on smithi145 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[97022]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[99461]: Upgrade: Updating osd.5 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[99461]: Deploying daemon osd.5 on smithi145 2024-09-13T14:26:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:31 smithi026 ceph-mon[99461]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2252: done (3 left) 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2253: read oid 28 snap -1 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2253: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2254: rollback oid 36 current snap is 209 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 204 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2251: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2251: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:2251: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:26:32.246 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1073 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:26:32.247 INFO:tasks.rados.rados.0.smithi026.stdout:2250: expect (ObjNum 687 snap 193 seq_num 687) 2024-09-13T14:26:32.304 INFO:tasks.rados.rados.0.smithi026.stdout:2249: expect (ObjNum 679 snap 192 seq_num 679) 2024-09-13T14:26:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:2254: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:26:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:2254: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:26:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:2254: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:26:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1168 (ObjNum 690 snap 195 seq_num 690) dirty exists 2024-09-13T14:26:32.587 INFO:tasks.rados.rados.0.smithi026.stdout:2253: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:32.616 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:32 smithi145 ceph-mon[89145]: pgmap v338: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 16 op/s 2024-09-13T14:26:32.616 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:32 smithi145 ceph-mon[89145]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:26:32.616 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:32 smithi145 ceph-mon[89145]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:26:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[99461]: pgmap v338: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 16 op/s 2024-09-13T14:26:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[99461]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:26:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[99461]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:26:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[97022]: pgmap v338: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 16 op/s 2024-09-13T14:26:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[97022]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:26:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:32 smithi026 ceph-mon[97022]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:26:32.666 INFO:tasks.rados.rados.0.smithi026.stdout:2249: done (4 left) 2024-09-13T14:26:32.666 INFO:tasks.rados.rados.0.smithi026.stdout:2250: done (3 left) 2024-09-13T14:26:32.666 INFO:tasks.rados.rados.0.smithi026.stdout:2251: done (2 left) 2024-09-13T14:26:32.666 INFO:tasks.rados.rados.0.smithi026.stdout:2253: done (1 left) 2024-09-13T14:26:32.666 INFO:tasks.rados.rados.0.smithi026.stdout:2254: done (0 left) 2024-09-13T14:26:32.667 INFO:tasks.rados.rados.0.smithi026.stdout:2255: snap_remove snap 202 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2255: done (0 left) 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2256: rollback oid 32 current snap is 209 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 204 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2257: read oid 17 snap -1 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2257: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2258: read oid 5 snap -1 2024-09-13T14:26:33.259 INFO:tasks.rados.rados.0.smithi026.stdout:2258: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:26:33.260 INFO:tasks.rados.rados.0.smithi026.stdout:2259: delete oid 25 current snap is 209 2024-09-13T14:26:33.286 INFO:tasks.rados.rados.0.smithi026.stdout:2259: done (3 left) 2024-09-13T14:26:33.286 INFO:tasks.rados.rados.0.smithi026.stdout:2260: delete oid 28 current snap is 209 2024-09-13T14:26:33.290 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:32 smithi145 systemd[1]: Stopping Ceph osd.5 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:26:33.294 INFO:tasks.rados.rados.0.smithi026.stdout:2256: finishing rollback tid 0 to smithi02683976-32 2024-09-13T14:26:33.294 INFO:tasks.rados.rados.0.smithi026.stdout:2256: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:26:33.294 INFO:tasks.rados.rados.0.smithi026.stdout:2256: finishing rollback tid 2 to smithi02683976-32 2024-09-13T14:26:33.295 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1037 (ObjNum 646 snap 182 seq_num 646) dirty exists 2024-09-13T14:26:33.295 INFO:tasks.rados.rados.0.smithi026.stdout:2256: done (3 left) 2024-09-13T14:26:33.295 INFO:tasks.rados.rados.0.smithi026.stdout:2260: done (2 left) 2024-09-13T14:26:33.295 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 209 2024-09-13T14:26:33.295 INFO:tasks.rados.rados.0.smithi026.stdout:2261: seq_num 716 ranges {6717440=794624,7512064=655360,8167424=704512,8871936=245760} 2024-09-13T14:26:33.337 INFO:tasks.rados.rados.0.smithi026.stdout:2261: writing smithi02683976-46 from 6717440 to 7512064 tid 1 2024-09-13T14:26:33.341 INFO:tasks.rados.rados.0.smithi026.stdout:2261: writing smithi02683976-46 from 7512064 to 8167424 tid 2 2024-09-13T14:26:33.345 INFO:tasks.rados.rados.0.smithi026.stdout:2261: writing smithi02683976-46 from 8167424 to 8871936 tid 3 2024-09-13T14:26:33.347 INFO:tasks.rados.rados.0.smithi026.stdout:2261: writing smithi02683976-46 from 8871936 to 9117696 tid 4 2024-09-13T14:26:33.347 INFO:tasks.rados.rados.0.smithi026.stdout:2262: rmattr oid 21 current snap is 209 2024-09-13T14:26:33.347 INFO:tasks.rados.rados.0.smithi026.stdout:2258: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:26:33.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[42486]: 2024-09-13T14:26:33.288+0000 7efe96bfd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:26:33.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[42486]: 2024-09-13T14:26:33.288+0000 7efe96bfd640 -1 osd.5 495 *** Got signal Terminated *** 2024-09-13T14:26:33.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[42486]: 2024-09-13T14:26:33.288+0000 7efe96bfd640 -1 osd.5 495 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:26:34.130 INFO:tasks.rados.rados.0.smithi026.stdout:2257: expect (ObjNum 578 snap 164 seq_num 578) 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[99461]: pgmap v341: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 29 op/s 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[99461]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[99461]: osd.5 marked itself down and dead 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[99461]: osdmap e496: 8 total, 7 up, 8 in 2024-09-13T14:26:34.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[97022]: pgmap v341: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 29 op/s 2024-09-13T14:26:34.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[97022]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:26:34.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[97022]: osd.5 marked itself down and dead 2024-09-13T14:26:34.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:34.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:34 smithi026 ceph-mon[97022]: osdmap e496: 8 total, 7 up, 8 in 2024-09-13T14:26:34.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:34 smithi145 podman[99634]: 2024-09-13 14:26:34.50438106 +0000 UTC m=+1.380426148 container died 9910270fa52911ec978983fabef7e7d5351a3e497f240edb3ea945420bddbeb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:26:34.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:34 smithi145 ceph-mon[89145]: pgmap v341: 121 pgs: 121 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 29 op/s 2024-09-13T14:26:34.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:34 smithi145 ceph-mon[89145]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:26:34.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:34 smithi145 ceph-mon[89145]: osd.5 marked itself down and dead 2024-09-13T14:26:34.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:34 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:26:34.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:34 smithi145 ceph-mon[89145]: osdmap e496: 8 total, 7 up, 8 in 2024-09-13T14:26:34.945 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 1 to smithi02683976-46 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 2 to smithi02683976-46 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2257: done (3 left) 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2258: done (2 left) 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 209 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2263: seq_num 717 ranges {0=16384} 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2263: writing smithi02683976-44 from 0 to 16384 tid 1 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2264: delete oid 19 current snap is 209 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 3 to smithi02683976-46 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 4 to smithi02683976-46 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 5 to smithi02683976-46 2024-09-13T14:26:34.946 INFO:tasks.rados.rados.0.smithi026.stdout:2261: finishing write tid 6 to smithi02683976-46 2024-09-13T14:26:34.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 931 (ObjNum 716 snap 209 seq_num 716) dirty exists 2024-09-13T14:26:34.947 INFO:tasks.rados.rados.0.smithi026.stdout:2261: left oid 46 (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:26:34.947 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1121 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:26:34.957 INFO:tasks.rados.rados.0.smithi026.stdout:2261: done (3 left) 2024-09-13T14:26:34.957 INFO:tasks.rados.rados.0.smithi026.stdout:2262: done (2 left) 2024-09-13T14:26:34.957 INFO:tasks.rados.rados.0.smithi026.stdout:2264: done (1 left) 2024-09-13T14:26:34.958 INFO:tasks.rados.rados.0.smithi026.stdout:2265: setattr oid 46 current snap is 209 2024-09-13T14:26:34.958 INFO:tasks.rados.rados.0.smithi026.stdout:2263: finishing write tid 1 to smithi02683976-44 2024-09-13T14:26:34.958 INFO:tasks.rados.rados.0.smithi026.stdout:2263: finishing write tid 2 to smithi02683976-44 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2266: read oid 2 snap -1 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2266: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2267: copy_from oid 7 from oid 21 current snap is 209 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2268: read oid 38 snap -1 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2268: expect (ObjNum 673 snap 189 seq_num 673) 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2269: snap_create 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:2263: finishing write tid 3 to smithi02683976-44 2024-09-13T14:26:34.960 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 933 (ObjNum 717 snap 209 seq_num 717) dirty exists 2024-09-13T14:26:34.961 INFO:tasks.rados.rados.0.smithi026.stdout:2263: left oid 44 (ObjNum 717 snap 209 seq_num 717) 2024-09-13T14:26:34.963 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 934 (ObjNum 716 snap 209 seq_num 716) dirty exists 2024-09-13T14:26:34.968 INFO:tasks.rados.rados.0.smithi026.stdout:2268: expect (ObjNum 673 snap 189 seq_num 673) 2024-09-13T14:26:35.013 INFO:tasks.rados.rados.0.smithi026.stdout:2266: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:26:35.069 INFO:tasks.rados.rados.0.smithi026.stdout:2267: finishing copy_from racing read to smithi02683976-7 2024-09-13T14:26:35.069 INFO:tasks.rados.rados.0.smithi026.stdout:2267: finishing copy_from to smithi02683976-7 2024-09-13T14:26:35.069 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1121 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:26:35.088 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:34 smithi145 podman[99634]: 2024-09-13 14:26:34.885767299 +0000 UTC m=+1.761812373 container cleanup 9910270fa52911ec978983fabef7e7d5351a3e497f240edb3ea945420bddbeb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:26:35.088 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:34 smithi145 bash[99634]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T14:26:35.455 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99654]: 2024-09-13 14:26:35.087428923 +0000 UTC m=+0.582664206 container remove 9910270fa52911ec978983fabef7e7d5351a3e497f240edb3ea945420bddbeb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:26:35.883 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.397060734 +0000 UTC m=+0.015909669 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:35.884 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.496867092 +0000 UTC m=+0.115716026 container create b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:26:35.884 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.724300521 +0000 UTC m=+0.343149454 container init b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:26:35.884 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.729503173 +0000 UTC m=+0.348352108 container start b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-13T14:26:35.949 INFO:tasks.rados.rados.0.smithi026.stdout:2263: done (5 left) 2024-09-13T14:26:35.949 INFO:tasks.rados.rados.0.smithi026.stdout:2265: done (4 left) 2024-09-13T14:26:35.949 INFO:tasks.rados.rados.0.smithi026.stdout:2266: done (3 left) 2024-09-13T14:26:35.949 INFO:tasks.rados.rados.0.smithi026.stdout:2267: done (2 left) 2024-09-13T14:26:35.950 INFO:tasks.rados.rados.0.smithi026.stdout:2268: done (1 left) 2024-09-13T14:26:35.950 INFO:tasks.rados.rados.0.smithi026.stdout:2269: done (0 left) 2024-09-13T14:26:35.950 INFO:tasks.rados.rados.0.smithi026.stdout:2270: snap_create 2024-09-13T14:26:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:35 smithi145 ceph-mon[89145]: pgmap v344: 121 pgs: 18 stale+active+clean, 103 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:26:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:35 smithi145 ceph-mon[89145]: osdmap e497: 8 total, 7 up, 8 in 2024-09-13T14:26:36.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.882543346 +0000 UTC m=+0.501392281 container attach b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T14:26:36.206 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:35 smithi145 podman[99751]: 2024-09-13 14:26:35.92963581 +0000 UTC m=+0.548484745 container died b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T14:26:36.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:35 smithi026 ceph-mon[99461]: pgmap v344: 121 pgs: 18 stale+active+clean, 103 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:26:36.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:35 smithi026 ceph-mon[99461]: osdmap e497: 8 total, 7 up, 8 in 2024-09-13T14:26:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:35 smithi026 ceph-mon[97022]: pgmap v344: 121 pgs: 18 stale+active+clean, 103 active+clean; 192 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:26:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:35 smithi026 ceph-mon[97022]: osdmap e497: 8 total, 7 up, 8 in 2024-09-13T14:26:36.951 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:36 smithi145 podman[99793]: 2024-09-13 14:26:36.628434237 +0000 UTC m=+0.689788948 container remove b6e791dcdc07e0007da4a6a78e4e81243b281873bf7b321a9f9c12b8a7b0faac (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0) 2024-09-13T14:26:36.951 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:36 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T14:26:36.951 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:36 smithi145 systemd[1]: Stopped Ceph osd.5 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:26:36.951 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:36 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.5.service: Consumed 16.101s CPU time. 2024-09-13T14:26:36.951 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:36 smithi145 systemd[1]: Starting Ceph osd.5 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:26:36.955 INFO:tasks.rados.rados.0.smithi026.stdout:2270: done (0 left) 2024-09-13T14:26:36.955 INFO:tasks.rados.rados.0.smithi026.stdout:2271: delete oid 45 current snap is 211 2024-09-13T14:26:36.957 INFO:tasks.rados.rados.0.smithi026.stdout:2271: done (0 left) 2024-09-13T14:26:36.957 INFO:tasks.rados.rados.0.smithi026.stdout:2272: read oid 40 snap -1 2024-09-13T14:26:36.957 INFO:tasks.rados.rados.0.smithi026.stdout:2272: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:36.957 INFO:tasks.rados.rados.0.smithi026.stdout:2273: snap_remove snap 209 2024-09-13T14:26:37.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:36 smithi145 ceph-mon[89145]: osdmap e498: 8 total, 7 up, 8 in 2024-09-13T14:26:37.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:36.973144706 +0000 UTC m=+0.016415879 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:37.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:37.088410969 +0000 UTC m=+0.131682199 container create 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:26:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:36 smithi026 ceph-mon[97022]: osdmap e498: 8 total, 7 up, 8 in 2024-09-13T14:26:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:36 smithi026 ceph-mon[99461]: osdmap e498: 8 total, 7 up, 8 in 2024-09-13T14:26:37.475 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:37.315767943 +0000 UTC m=+0.359039109 container init 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:26:37.475 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:37.321097358 +0000 UTC m=+0.364368526 container start 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T14:26:37.475 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:37.423981953 +0000 UTC m=+0.467253122 container attach 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:26:37.476 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.476 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.476 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.476 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-13T14:26:37.867 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-13T14:26:37.868 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:26:37.868 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:26:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2273: done (1 left) 2024-09-13T14:26:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 211 2024-09-13T14:26:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2274: seq_num 720 ranges {1966080=65536} 2024-09-13T14:26:37.967 INFO:tasks.rados.rados.0.smithi026.stdout:2274: writing smithi02683976-36 from 1966080 to 2031616 tid 1 2024-09-13T14:26:37.967 INFO:tasks.rados.rados.0.smithi026.stdout:2275: delete oid 1 current snap is 211 2024-09-13T14:26:37.968 INFO:tasks.rados.rados.0.smithi026.stdout:2272: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:38.146 INFO:tasks.rados.rados.0.smithi026.stdout:2274: finishing write tid 1 to smithi02683976-36 2024-09-13T14:26:38.146 INFO:tasks.rados.rados.0.smithi026.stdout:2274: finishing write tid 2 to smithi02683976-36 2024-09-13T14:26:38.146 INFO:tasks.rados.rados.0.smithi026.stdout:2274: finishing write tid 3 to smithi02683976-36 2024-09-13T14:26:38.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1175 (ObjNum 720 snap 211 seq_num 720) dirty exists 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2274: left oid 36 (ObjNum 720 snap 211 seq_num 720) 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2272: done (2 left) 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2274: done (1 left) 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2275: done (0 left) 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2276: read oid 34 snap -1 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2276: expect (ObjNum 672 snap 189 seq_num 672) 2024-09-13T14:26:38.147 INFO:tasks.rados.rados.0.smithi026.stdout:2277: delete oid 41 current snap is 211 2024-09-13T14:26:38.189 INFO:tasks.rados.rados.0.smithi026.stdout:2277: done (1 left) 2024-09-13T14:26:38.189 INFO:tasks.rados.rados.0.smithi026.stdout:2278: delete oid 44 current snap is 211 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2278: done (1 left) 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2279: read oid 32 snap 204 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2279: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2280: read oid 4 snap -1 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2280: expect deleted 2024-09-13T14:26:38.194 INFO:tasks.rados.rados.0.smithi026.stdout:2281: delete oid 17 current snap is 211 2024-09-13T14:26:38.199 INFO:tasks.rados.rados.0.smithi026.stdout:2276: expect (ObjNum 672 snap 189 seq_num 672) 2024-09-13T14:26:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-mon[89145]: pgmap v347: 121 pgs: 23 active+undersized, 7 stale+active+clean, 12 active+undersized+degraded, 79 active+clean; 194 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 590 KiB/s wr, 17 op/s; 37/255 objects degraded (14.510%) 2024-09-13T14:26:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 37/255 objects degraded (14.510%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-mon[89145]: osdmap e499: 8 total, 7 up, 8 in 2024-09-13T14:26:38.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:26:38.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:26:38.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:26:38.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:26:38.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:26:38.206 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:26:38.206 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate[99913]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-13T14:26:38.206 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 bash[99898]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-13T14:26:38.206 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:37 smithi145 podman[99898]: 2024-09-13 14:26:37.895646717 +0000 UTC m=+0.938917883 container died 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:26:38.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[99461]: pgmap v347: 121 pgs: 23 active+undersized, 7 stale+active+clean, 12 active+undersized+degraded, 79 active+clean; 194 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 590 KiB/s wr, 17 op/s; 37/255 objects degraded (14.510%) 2024-09-13T14:26:38.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 37/255 objects degraded (14.510%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:38.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[99461]: osdmap e499: 8 total, 7 up, 8 in 2024-09-13T14:26:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[97022]: pgmap v347: 121 pgs: 23 active+undersized, 7 stale+active+clean, 12 active+undersized+degraded, 79 active+clean; 194 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 590 KiB/s wr, 17 op/s; 37/255 objects degraded (14.510%) 2024-09-13T14:26:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 37/255 objects degraded (14.510%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:37 smithi026 ceph-mon[97022]: osdmap e499: 8 total, 7 up, 8 in 2024-09-13T14:26:38.955 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:38 smithi145 podman[100063]: 2024-09-13 14:26:38.636958948 +0000 UTC m=+0.732170776 container remove 758fc27d3adab58debcc93be714236624282457b73183b9124dc19ed16614f8a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T14:26:38.955 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:38 smithi145 podman[100137]: 2024-09-13 14:26:38.744887775 +0000 UTC m=+0.016040854 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:26:38.955 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:38 smithi145 podman[100137]: 2024-09-13 14:26:38.871389924 +0000 UTC m=+0.142543003 container create b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T14:26:39.196 INFO:tasks.rados.rados.0.smithi026.stdout:2279: expect (ObjNum 646 snap 182 seq_num 646) 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2276: done (3 left) 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2279: done (2 left) 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2280: done (1 left) 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2281: done (0 left) 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2282: rollback oid 1 current snap is 211 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 210 2024-09-13T14:26:39.283 INFO:tasks.rados.rados.0.smithi026.stdout:2283: snap_create 2024-09-13T14:26:39.285 INFO:tasks.rados.rados.0.smithi026.stdout:2282: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:26:39.285 INFO:tasks.rados.rados.0.smithi026.stdout:2282: finishing rollback tid 2 to smithi02683976-1 2024-09-13T14:26:39.285 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1179 (ObjNum 616 snap 176 seq_num 616) dirty exists 2024-09-13T14:26:39.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:38 smithi026 ceph-mon[97022]: osdmap e500: 8 total, 7 up, 8 in 2024-09-13T14:26:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:38 smithi026 ceph-mon[97022]: osdmap e501: 8 total, 7 up, 8 in 2024-09-13T14:26:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:38 smithi026 ceph-mon[99461]: osdmap e500: 8 total, 7 up, 8 in 2024-09-13T14:26:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:38 smithi026 ceph-mon[99461]: osdmap e501: 8 total, 7 up, 8 in 2024-09-13T14:26:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:38 smithi145 ceph-mon[89145]: osdmap e500: 8 total, 7 up, 8 in 2024-09-13T14:26:39.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:38 smithi145 ceph-mon[89145]: osdmap e501: 8 total, 7 up, 8 in 2024-09-13T14:26:39.456 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:39 smithi145 podman[100137]: 2024-09-13 14:26:39.098856156 +0000 UTC m=+0.370009234 container init b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T14:26:39.456 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:39 smithi145 podman[100137]: 2024-09-13 14:26:39.103624981 +0000 UTC m=+0.374778062 container start b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T14:26:39.457 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:39 smithi145 ceph-osd[100175]: -- 172.21.15.145:0/1129458787 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556bf2571a40 con 0x556bf3340400 2024-09-13T14:26:39.457 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:39 smithi145 bash[100137]: b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 2024-09-13T14:26:39.457 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:39 smithi145 systemd[1]: Started Ceph osd.5 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:26:39.570 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:26:40.153 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 64s ago 10m 18.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (4m) 30s ago 9m 87.1M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (11m) 64s ago 11m 15.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (11m) 30s ago 11m 22.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (4m) 30s ago 14m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (4m) 64s ago 16m 523M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 64s ago 16m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (3m) 30s ago 15m 44.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 64s ago 15m 46.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:26:40.154 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 64s ago 9m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (4m) 30s ago 9m 11.4M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 64s ago 14m 152M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 64s ago 14m 144M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (101s) 64s ago 13m 108M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (68s) 64s ago 13m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (35s) 30s ago 13m 14.5M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 starting - - - 2480M 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (12m) 30s ago 12m 153M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (11m) 30s ago 11m 164M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:26:40.155 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (4m) 30s ago 10m 49.7M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:26:40.257 INFO:tasks.rados.rados.0.smithi026.stdout:2282: done (1 left) 2024-09-13T14:26:40.257 INFO:tasks.rados.rados.0.smithi026.stdout:2283: done (0 left) 2024-09-13T14:26:40.258 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 212 2024-09-13T14:26:40.258 INFO:tasks.rados.rados.0.smithi026.stdout:2284: seq_num 725 ranges {3129344=147456} 2024-09-13T14:26:40.275 INFO:tasks.rados.rados.0.smithi026.stdout:2284: writing smithi02683976-20 from 3129344 to 3276800 tid 1 2024-09-13T14:26:40.275 INFO:tasks.rados.rados.0.smithi026.stdout:2285: delete oid 11 current snap is 212 2024-09-13T14:26:40.280 INFO:tasks.rados.rados.0.smithi026.stdout:2285: done (1 left) 2024-09-13T14:26:40.280 INFO:tasks.rados.rados.0.smithi026.stdout:2286: snap_remove snap 204 2024-09-13T14:26:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[97022]: pgmap v350: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 38/258 objects degraded (14.729%) 2024-09-13T14:26:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[99461]: pgmap v350: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 38/258 objects degraded (14.729%) 2024-09-13T14:26:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:40 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2, 2024-09-13T14:26:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 5 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 10 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:26:40.513 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:26:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-mon[89145]: pgmap v350: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 19 op/s; 38/258 objects degraded (14.729%) 2024-09-13T14:26:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:40.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:40.808 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:26:40.808 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:26:40.808 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "10/20 daemons upgraded", 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:26:40.809 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:26:41.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:26:40.754+0000 7f35a1609640 -1 Falling back to public interface 2024-09-13T14:26:41.263 INFO:tasks.rados.rados.0.smithi026.stdout:2286: done (1 left) 2024-09-13T14:26:41.263 INFO:tasks.rados.rados.0.smithi026.stdout:2284: finishing write tid 1 to smithi02683976-20 2024-09-13T14:26:41.263 INFO:tasks.rados.rados.0.smithi026.stdout:2284: finishing write tid 2 to smithi02683976-20 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2284: finishing write tid 3 to smithi02683976-20 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1085 (ObjNum 725 snap 212 seq_num 725) dirty exists 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2284: left oid 20 (ObjNum 725 snap 212 seq_num 725) 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2287: read oid 18 snap 210 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2287: expect deleted 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2284: done (1 left) 2024-09-13T14:26:41.264 INFO:tasks.rados.rados.0.smithi026.stdout:2288: delete oid 12 current snap is 212 2024-09-13T14:26:41.268 INFO:tasks.rados.rados.0.smithi026.stdout:2288: done (1 left) 2024-09-13T14:26:41.268 INFO:tasks.rados.rados.0.smithi026.stdout:2289: read oid 47 snap -1 2024-09-13T14:26:41.268 INFO:tasks.rados.rados.0.smithi026.stdout:2289: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:41.268 INFO:tasks.rados.rados.0.smithi026.stdout:2290: snap_create 2024-09-13T14:26:41.286 INFO:tasks.rados.rados.0.smithi026.stdout:2289: expect (ObjNum 693 snap 195 seq_num 693) 2024-09-13T14:26:41.642 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.642 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.642 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.642 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: osdmap e502: 8 total, 7 up, 8 in 2024-09-13T14:26:41.642 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/697492490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:41.643 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:41.643 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:41] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.51.0" 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: osdmap e502: 8 total, 7 up, 8 in 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/697492490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: osdmap e502: 8 total, 7 up, 8 in 2024-09-13T14:26:41.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/697492490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:41.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:41.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2287: done (2 left) 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2289: done (1 left) 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2290: done (0 left) 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2291: read oid 46 snap 212 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2291: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:26:42.268 INFO:tasks.rados.rados.0.smithi026.stdout:2292: copy_from oid 8 from oid 16 current snap is 213 2024-09-13T14:26:42.269 INFO:tasks.rados.rados.0.smithi026.stdout:2293: read oid 4 snap -1 2024-09-13T14:26:42.269 INFO:tasks.rados.rados.0.smithi026.stdout:2293: expect deleted 2024-09-13T14:26:42.269 INFO:tasks.rados.rados.0.smithi026.stdout:2294: snap_create 2024-09-13T14:26:42.312 INFO:tasks.rados.rados.0.smithi026.stdout:2291: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:26:42.523 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:42 smithi145 ceph-mon[89145]: pgmap v353: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s; 41/273 objects degraded (15.018%) 2024-09-13T14:26:42.524 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:42 smithi145 ceph-mon[89145]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:42.524 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:42 smithi145 ceph-mon[89145]: osdmap e503: 8 total, 7 up, 8 in 2024-09-13T14:26:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[97022]: pgmap v353: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s; 41/273 objects degraded (15.018%) 2024-09-13T14:26:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[97022]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[97022]: osdmap e503: 8 total, 7 up, 8 in 2024-09-13T14:26:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[99461]: pgmap v353: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s; 41/273 objects degraded (15.018%) 2024-09-13T14:26:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[99461]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:42 smithi026 ceph-mon[99461]: osdmap e503: 8 total, 7 up, 8 in 2024-09-13T14:26:42.772 INFO:tasks.rados.rados.0.smithi026.stdout:2292: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:26:42.772 INFO:tasks.rados.rados.0.smithi026.stdout:2292: finishing copy_from to smithi02683976-8 2024-09-13T14:26:42.772 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 979 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:26:43.272 INFO:tasks.rados.rados.0.smithi026.stdout:2291: done (3 left) 2024-09-13T14:26:43.272 INFO:tasks.rados.rados.0.smithi026.stdout:2292: done (2 left) 2024-09-13T14:26:43.273 INFO:tasks.rados.rados.0.smithi026.stdout:2293: done (1 left) 2024-09-13T14:26:43.273 INFO:tasks.rados.rados.0.smithi026.stdout:2294: done (0 left) 2024-09-13T14:26:43.273 INFO:tasks.rados.rados.0.smithi026.stdout:2295: delete oid 16 current snap is 214 2024-09-13T14:26:43.278 INFO:tasks.rados.rados.0.smithi026.stdout:2295: done (0 left) 2024-09-13T14:26:43.278 INFO:tasks.rados.rados.0.smithi026.stdout:2296: rollback oid 28 current snap is 214 2024-09-13T14:26:43.278 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 213 2024-09-13T14:26:43.279 INFO:tasks.rados.rados.0.smithi026.stdout:2297: copy_from oid 35 from oid 21 current snap is 214 2024-09-13T14:26:43.279 INFO:tasks.rados.rados.0.smithi026.stdout:2298: delete oid 50 current snap is 214 2024-09-13T14:26:43.280 INFO:tasks.rados.rados.0.smithi026.stdout:2296: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:26:43.280 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 0 (ObjNum 673 snap 0 seq_num 519571792) dirty dne 2024-09-13T14:26:43.282 INFO:tasks.rados.rados.0.smithi026.stdout:2296: done (2 left) 2024-09-13T14:26:43.282 INFO:tasks.rados.rados.0.smithi026.stdout:2298: done (1 left) 2024-09-13T14:26:43.283 INFO:tasks.rados.rados.0.smithi026.stdout:2299: setattr oid 39 current snap is 214 2024-09-13T14:26:43.284 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 214 2024-09-13T14:26:43.284 INFO:tasks.rados.rados.0.smithi026.stdout:2300: seq_num 730 ranges {7725056=229376} 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:2300: writing smithi02683976-1 from 7725056 to 7954432 tid 1 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:2301: read oid 12 snap -1 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:2301: expect deleted 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 214 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 989 (ObjNum 662 snap 188 seq_num 662) dirty exists 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:2297: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:26:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:2297: finishing copy_from to smithi02683976-35 2024-09-13T14:26:43.329 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1121 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:26:43.329 INFO:tasks.rados.rados.0.smithi026.stdout:2302: seq_num 731 ranges {2359296=245760} 2024-09-13T14:26:43.342 INFO:tasks.rados.rados.0.smithi026.stdout:2302: writing smithi02683976-32 from 2359296 to 2605056 tid 1 2024-09-13T14:26:43.342 INFO:tasks.rados.rados.0.smithi026.stdout:2297: done (4 left) 2024-09-13T14:26:43.342 INFO:tasks.rados.rados.0.smithi026.stdout:2299: done (3 left) 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:2303: snap_create 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:2300: finishing write tid 1 to smithi02683976-1 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:2300: finishing write tid 2 to smithi02683976-1 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:2300: finishing write tid 3 to smithi02683976-1 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1184 (ObjNum 730 snap 214 seq_num 730) dirty exists 2024-09-13T14:26:43.343 INFO:tasks.rados.rados.0.smithi026.stdout:2300: left oid 1 (ObjNum 730 snap 214 seq_num 730) 2024-09-13T14:26:43.344 INFO:tasks.rados.rados.0.smithi026.stdout:2302: finishing write tid 1 to smithi02683976-32 2024-09-13T14:26:43.344 INFO:tasks.rados.rados.0.smithi026.stdout:2302: finishing write tid 2 to smithi02683976-32 2024-09-13T14:26:43.345 INFO:tasks.rados.rados.0.smithi026.stdout:2302: finishing write tid 3 to smithi02683976-32 2024-09-13T14:26:43.345 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1040 (ObjNum 731 snap 214 seq_num 731) dirty exists 2024-09-13T14:26:43.345 INFO:tasks.rados.rados.0.smithi026.stdout:2302: left oid 32 (ObjNum 731 snap 214 seq_num 731) 2024-09-13T14:26:43.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:43 smithi026 ceph-mon[97022]: osdmap e504: 8 total, 7 up, 8 in 2024-09-13T14:26:43.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:43 smithi026 ceph-mon[99461]: osdmap e504: 8 total, 7 up, 8 in 2024-09-13T14:26:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:43 smithi145 ceph-mon[89145]: osdmap e504: 8 total, 7 up, 8 in 2024-09-13T14:26:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:2300: done (3 left) 2024-09-13T14:26:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:2301: done (2 left) 2024-09-13T14:26:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:2302: done (1 left) 2024-09-13T14:26:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:2303: done (0 left) 2024-09-13T14:26:43.945 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 215 2024-09-13T14:26:43.946 INFO:tasks.rados.rados.0.smithi026.stdout:2304: seq_num 732 ranges {0=262144} 2024-09-13T14:26:43.946 INFO:tasks.rados.rados.0.smithi026.stdout:2304: writing smithi02683976-6 from 0 to 262144 tid 1 2024-09-13T14:26:43.946 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 215 2024-09-13T14:26:43.946 INFO:tasks.rados.rados.0.smithi026.stdout:2305: seq_num 733 ranges {2031616=278528} 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2305: writing smithi02683976-36 from 2031616 to 2310144 tid 1 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2306: snap_create 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2304: finishing write tid 1 to smithi02683976-6 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2304: finishing write tid 2 to smithi02683976-6 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2304: finishing write tid 3 to smithi02683976-6 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1134 (ObjNum 732 snap 215 seq_num 732) dirty exists 2024-09-13T14:26:43.960 INFO:tasks.rados.rados.0.smithi026.stdout:2304: left oid 6 (ObjNum 732 snap 215 seq_num 732) 2024-09-13T14:26:43.966 INFO:tasks.rados.rados.0.smithi026.stdout:2305: finishing write tid 1 to smithi02683976-36 2024-09-13T14:26:43.966 INFO:tasks.rados.rados.0.smithi026.stdout:2305: finishing write tid 2 to smithi02683976-36 2024-09-13T14:26:43.967 INFO:tasks.rados.rados.0.smithi026.stdout:2305: finishing write tid 3 to smithi02683976-36 2024-09-13T14:26:43.967 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1187 (ObjNum 733 snap 215 seq_num 733) dirty exists 2024-09-13T14:26:43.967 INFO:tasks.rados.rados.0.smithi026.stdout:2305: left oid 36 (ObjNum 733 snap 215 seq_num 733) 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: pgmap v356: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 204 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 52 KiB/s wr, 6 op/s; 42/264 objects degraded (15.909%) 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 42/264 objects degraded (15.909%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: osdmap e505: 8 total, 7 up, 8 in 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[99461]: osdmap e506: 8 total, 7 up, 8 in 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: pgmap v356: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 204 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 52 KiB/s wr, 6 op/s; 42/264 objects degraded (15.909%) 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 42/264 objects degraded (15.909%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: osdmap e505: 8 total, 7 up, 8 in 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:44 smithi026 ceph-mon[97022]: osdmap e506: 8 total, 7 up, 8 in 2024-09-13T14:26:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: pgmap v356: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 204 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 52 KiB/s wr, 6 op/s; 42/264 objects degraded (15.909%) 2024-09-13T14:26:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 42/264 objects degraded (15.909%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: osdmap e505: 8 total, 7 up, 8 in 2024-09-13T14:26:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:44.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-mon[89145]: osdmap e506: 8 total, 7 up, 8 in 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2304: done (2 left) 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2305: done (1 left) 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2306: done (0 left) 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2307: rmattr oid 11 current snap is 216 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2307: done (0 left) 2024-09-13T14:26:44.948 INFO:tasks.rados.rados.0.smithi026.stdout:2308: read oid 45 snap -1 2024-09-13T14:26:44.949 INFO:tasks.rados.rados.0.smithi026.stdout:2308: expect deleted 2024-09-13T14:26:44.949 INFO:tasks.rados.rados.0.smithi026.stdout:2309: copy_from oid 30 from oid 9 current snap is 216 2024-09-13T14:26:44.949 INFO:tasks.rados.rados.0.smithi026.stdout:2310: snap_create 2024-09-13T14:26:44.965 INFO:tasks.rados.rados.0.smithi026.stdout:2309: finishing copy_from racing read to smithi02683976-30 2024-09-13T14:26:44.967 INFO:tasks.rados.rados.0.smithi026.stdout:2309: finishing copy_from to smithi02683976-30 2024-09-13T14:26:44.967 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1025 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:26:45.043 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:26:44.782+0000 7f35a1609640 -1 osd.5 495 log_to_monitors true 2024-09-13T14:26:45.043 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:26:44 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:26:44.970+0000 7f35993b3640 -1 osd.5 495 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:26:45.684 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.684 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: pgmap v359: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 36 KiB/s wr, 1 op/s; 42/261 objects degraded (16.092%) 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: osdmap e507: 8 total, 7 up, 8 in 2024-09-13T14:26:45.685 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:45 smithi145 ceph-mon[89145]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: pgmap v359: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 36 KiB/s wr, 1 op/s; 42/261 objects degraded (16.092%) 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: osdmap e507: 8 total, 7 up, 8 in 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[97022]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: pgmap v359: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 36 KiB/s wr, 1 op/s; 42/261 objects degraded (16.092%) 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: osdmap e507: 8 total, 7 up, 8 in 2024-09-13T14:26:45.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:45 smithi026 ceph-mon[99461]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2308: done (2 left) 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2309: done (1 left) 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2310: done (0 left) 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2311: rollback oid 30 current snap is 217 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 215 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2312: rollback oid 36 current snap is 217 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 215 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2313: read oid 1 snap -1 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2313: expect (ObjNum 730 snap 214 seq_num 730) 2024-09-13T14:26:45.954 INFO:tasks.rados.rados.0.smithi026.stdout:2314: snap_remove snap 212 2024-09-13T14:26:46.571 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:46 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:46.572 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:46 smithi145 ceph-mon[89145]: osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676] boot 2024-09-13T14:26:46.572 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:46 smithi145 ceph-mon[89145]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:26:46.572 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:46 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:26:46.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[99461]: osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676] boot 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[99461]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[97022]: osd.5 [v2:172.21.15.145:6808/313394676,v1:172.21.15.145:6809/313394676] boot 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[97022]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:26:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:46 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:26:46.951 INFO:tasks.rados.rados.0.smithi026.stdout:2314: done (3 left) 2024-09-13T14:26:46.951 INFO:tasks.rados.rados.0.smithi026.stdout:2315: snap_create 2024-09-13T14:26:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2311: finishing rollback tid 0 to smithi02683976-30 2024-09-13T14:26:46.994 INFO:tasks.rados.rados.0.smithi026.stdout:2311: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:26:46.994 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1027 (ObjNum 65 snap 0 seq_num 1819303513) dirty dne 2024-09-13T14:26:47.008 INFO:tasks.rados.rados.0.smithi026.stdout:2313: expect (ObjNum 730 snap 214 seq_num 730) 2024-09-13T14:26:47.864 INFO:tasks.rados.rados.0.smithi026.stdout:2312: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:26:47.864 INFO:tasks.rados.rados.0.smithi026.stdout:2312: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:26:47.865 INFO:tasks.rados.rados.0.smithi026.stdout:2312: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:26:47.865 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1191 (ObjNum 733 snap 215 seq_num 733) dirty exists 2024-09-13T14:26:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:2311: done (3 left) 2024-09-13T14:26:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:2312: done (2 left) 2024-09-13T14:26:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:2313: done (1 left) 2024-09-13T14:26:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:2315: done (0 left) 2024-09-13T14:26:47.951 INFO:tasks.rados.rados.0.smithi026.stdout:2316: setattr oid 22 current snap is 218 2024-09-13T14:26:47.952 INFO:tasks.rados.rados.0.smithi026.stdout:2317: snap_remove snap 217 2024-09-13T14:26:48.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:26:47.912+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: pgmap v362: 121 pgs: 17 peering, 25 active+undersized, 7 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.2 MiB/s wr, 6 op/s; 34/270 objects degraded (12.593%) 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: pgmap v362: 121 pgs: 17 peering, 25 active+undersized, 7 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.2 MiB/s wr, 6 op/s; 34/270 objects degraded (12.593%) 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:47 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: pgmap v362: 121 pgs: 17 peering, 25 active+undersized, 7 active+undersized+degraded, 72 active+clean; 206 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.2 MiB/s wr, 6 op/s; 34/270 objects degraded (12.593%) 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:26:48.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:48.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:48.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:26:48.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:47 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:26:48.962 INFO:tasks.rados.rados.0.smithi026.stdout:2317: done (1 left) 2024-09-13T14:26:48.963 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 218 2024-09-13T14:26:48.963 INFO:tasks.rados.rados.0.smithi026.stdout:2318: seq_num 734 ranges {0=294912} 2024-09-13T14:26:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:2318: writing smithi02683976-11 from 0 to 294912 tid 1 2024-09-13T14:26:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 218 2024-09-13T14:26:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 971 (ObjNum 691 snap 195 seq_num 691) dirty exists 2024-09-13T14:26:48.964 INFO:tasks.rados.rados.0.smithi026.stdout:2319: seq_num 735 ranges {540672=311296} 2024-09-13T14:26:48.969 INFO:tasks.rados.rados.0.smithi026.stdout:2319: writing smithi02683976-26 from 540672 to 851968 tid 1 2024-09-13T14:26:48.969 INFO:tasks.rados.rados.0.smithi026.stdout:2316: done (2 left) 2024-09-13T14:26:48.969 INFO:tasks.rados.rados.0.smithi026.stdout:2320: setattr oid 23 current snap is 218 2024-09-13T14:26:48.970 INFO:tasks.rados.rados.0.smithi026.stdout:2318: finishing write tid 1 to smithi02683976-11 2024-09-13T14:26:48.970 INFO:tasks.rados.rados.0.smithi026.stdout:2318: finishing write tid 2 to smithi02683976-11 2024-09-13T14:26:48.970 INFO:tasks.rados.rados.0.smithi026.stdout:2321: snap_remove snap 216 2024-09-13T14:26:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[97022]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 34/270 objects degraded (12.593%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[99461]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:26:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:48 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 34/270 objects degraded (12.593%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:48 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:26:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:48 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-13T14:26:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:48 smithi145 ceph-mon[89145]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:26:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:48 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 34/270 objects degraded (12.593%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2321: done (3 left) 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2322: read oid 4 snap -1 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2322: expect deleted 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2323: delete oid 14 current snap is 218 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2318: finishing write tid 3 to smithi02683976-11 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1138 (ObjNum 734 snap 218 seq_num 734) dirty exists 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2318: left oid 11 (ObjNum 734 snap 218 seq_num 734) 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1048 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2319: finishing write tid 1 to smithi02683976-26 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2319: finishing write tid 2 to smithi02683976-26 2024-09-13T14:26:49.970 INFO:tasks.rados.rados.0.smithi026.stdout:2319: finishing write tid 3 to smithi02683976-26 2024-09-13T14:26:49.971 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 973 (ObjNum 735 snap 218 seq_num 735) dirty exists 2024-09-13T14:26:49.971 INFO:tasks.rados.rados.0.smithi026.stdout:2319: left oid 26 (ObjNum 735 snap 218 seq_num 735) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2318: done (4 left) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2319: done (3 left) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2320: done (2 left) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2322: done (1 left) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2323: done (0 left) 2024-09-13T14:26:49.977 INFO:tasks.rados.rados.0.smithi026.stdout:2324: snap_remove snap 210 2024-09-13T14:26:50.213 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:49 smithi145 ceph-mon[89145]: pgmap v365: 121 pgs: 17 peering, 19 active+undersized, 6 active+undersized+degraded, 79 active+clean; 213 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 11 op/s; 33/276 objects degraded (11.957%) 2024-09-13T14:26:50.213 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:49 smithi145 ceph-mon[89145]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:26:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:49 smithi026 ceph-mon[97022]: pgmap v365: 121 pgs: 17 peering, 19 active+undersized, 6 active+undersized+degraded, 79 active+clean; 213 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 11 op/s; 33/276 objects degraded (11.957%) 2024-09-13T14:26:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:49 smithi026 ceph-mon[97022]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:26:50.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:49 smithi026 ceph-mon[99461]: pgmap v365: 121 pgs: 17 peering, 19 active+undersized, 6 active+undersized+degraded, 79 active+clean; 213 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 11 op/s; 33/276 objects degraded (11.957%) 2024-09-13T14:26:50.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:49 smithi026 ceph-mon[99461]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:26:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:26:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:26:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:50.976 INFO:tasks.rados.rados.0.smithi026.stdout:2324: done (0 left) 2024-09-13T14:26:50.977 INFO:tasks.rados.rados.0.smithi026.stdout:2325: rollback oid 11 current snap is 218 2024-09-13T14:26:50.977 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 215 2024-09-13T14:26:50.977 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 218 2024-09-13T14:26:50.977 INFO:tasks.rados.rados.0.smithi026.stdout:2326: seq_num 737 ranges {262144=352256} 2024-09-13T14:26:50.980 INFO:tasks.rados.rados.0.smithi026.stdout:2326: writing smithi02683976-6 from 262144 to 614400 tid 1 2024-09-13T14:26:50.980 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 218 2024-09-13T14:26:50.981 INFO:tasks.rados.rados.0.smithi026.stdout:2327: seq_num 738 ranges {0=368640} 2024-09-13T14:26:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2327: writing smithi02683976-37 from 0 to 368640 tid 1 2024-09-13T14:26:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2328: snap_create 2024-09-13T14:26:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2325: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:26:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:2325: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:26:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1142 (ObjNum 65 snap 0 seq_num 1819303513) dirty dne 2024-09-13T14:26:50.988 INFO:tasks.rados.rados.0.smithi026.stdout:2326: finishing write tid 1 to smithi02683976-6 2024-09-13T14:26:50.989 INFO:tasks.rados.rados.0.smithi026.stdout:2326: finishing write tid 2 to smithi02683976-6 2024-09-13T14:26:50.990 INFO:tasks.rados.rados.0.smithi026.stdout:2327: finishing write tid 1 to smithi02683976-37 2024-09-13T14:26:50.991 INFO:tasks.rados.rados.0.smithi026.stdout:2326: finishing write tid 3 to smithi02683976-6 2024-09-13T14:26:50.991 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1145 (ObjNum 737 snap 218 seq_num 737) dirty exists 2024-09-13T14:26:50.991 INFO:tasks.rados.rados.0.smithi026.stdout:2326: left oid 6 (ObjNum 737 snap 218 seq_num 737) 2024-09-13T14:26:50.991 INFO:tasks.rados.rados.0.smithi026.stdout:2327: finishing write tid 2 to smithi02683976-37 2024-09-13T14:26:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:2327: finishing write tid 3 to smithi02683976-37 2024-09-13T14:26:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1021 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:26:50.992 INFO:tasks.rados.rados.0.smithi026.stdout:2327: left oid 37 (ObjNum 738 snap 218 seq_num 738) 2024-09-13T14:26:51.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:50 smithi026 ceph-mon[97022]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:26:51.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:50 smithi026 ceph-mon[99461]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:26:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:50 smithi145 ceph-mon[89145]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:26:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:26:51] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.51.0" 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2325: done (3 left) 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2326: done (2 left) 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2327: done (1 left) 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2328: done (0 left) 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2329: rollback oid 16 current snap is 219 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 211 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2330: read oid 42 snap -1 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2330: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:51.981 INFO:tasks.rados.rados.0.smithi026.stdout:2331: snap_remove snap 214 2024-09-13T14:26:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[97022]: pgmap v368: 121 pgs: 17 peering, 104 active+clean; 200 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.8 MiB/s wr, 17 op/s; 13 MiB/s, 4 objects/s recovering 2024-09-13T14:26:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/276 objects degraded (11.957%), 6 pgs degraded) 2024-09-13T14:26:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:26:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[97022]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:26:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[99461]: pgmap v368: 121 pgs: 17 peering, 104 active+clean; 200 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.8 MiB/s wr, 17 op/s; 13 MiB/s, 4 objects/s recovering 2024-09-13T14:26:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/276 objects degraded (11.957%), 6 pgs degraded) 2024-09-13T14:26:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:26:52.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:51 smithi026 ceph-mon[99461]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:26:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:51 smithi145 ceph-mon[89145]: pgmap v368: 121 pgs: 17 peering, 104 active+clean; 200 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.8 MiB/s wr, 17 op/s; 13 MiB/s, 4 objects/s recovering 2024-09-13T14:26:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:51 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/276 objects degraded (11.957%), 6 pgs degraded) 2024-09-13T14:26:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:51 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:26:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:51 smithi145 ceph-mon[89145]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:2331: done (2 left) 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:2332: snap_create 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:2329: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:2329: finishing rollback tid 2 to smithi02683976-16 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1101 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:26:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:2330: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:26:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:52 smithi026 ceph-mon[97022]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:26:53.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:52 smithi026 ceph-mon[99461]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:26:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:52 smithi145 ceph-mon[89145]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:26:53.948 INFO:tasks.rados.rados.0.smithi026.stdout:2329: done (2 left) 2024-09-13T14:26:53.948 INFO:tasks.rados.rados.0.smithi026.stdout:2330: done (1 left) 2024-09-13T14:26:53.948 INFO:tasks.rados.rados.0.smithi026.stdout:2332: done (0 left) 2024-09-13T14:26:53.949 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 220 2024-09-13T14:26:53.949 INFO:tasks.rados.rados.0.smithi026.stdout:2333: seq_num 739 ranges {4571136=385024} 2024-09-13T14:26:53.975 INFO:tasks.rados.rados.0.smithi026.stdout:2333: writing smithi02683976-40 from 4571136 to 4956160 tid 1 2024-09-13T14:26:53.975 INFO:tasks.rados.rados.0.smithi026.stdout:2334: copy_from oid 27 from oid 9 current snap is 220 2024-09-13T14:26:53.975 INFO:tasks.rados.rados.0.smithi026.stdout:2335: snap_remove snap 213 2024-09-13T14:26:54.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[97022]: pgmap v371: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 234 KiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2024-09-13T14:26:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[97022]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:26:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[97022]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:26:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[99461]: pgmap v371: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 234 KiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2024-09-13T14:26:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[99461]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:26:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:53 smithi026 ceph-mon[99461]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:26:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:53 smithi145 ceph-mon[89145]: pgmap v371: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 234 KiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2024-09-13T14:26:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:53 smithi145 ceph-mon[89145]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:26:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:53 smithi145 ceph-mon[89145]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:26:54.949 INFO:tasks.rados.rados.0.smithi026.stdout:2335: done (2 left) 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2336: copy_from oid 32 from oid 2 current snap is 220 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2333: finishing write tid 1 to smithi02683976-40 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2333: finishing write tid 2 to smithi02683976-40 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2333: finishing write tid 3 to smithi02683976-40 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 955 (ObjNum 739 snap 220 seq_num 739) dirty exists 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2333: left oid 40 (ObjNum 739 snap 220 seq_num 739) 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2334: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2334: finishing copy_from to smithi02683976-27 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1025 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2333: done (2 left) 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2334: done (1 left) 2024-09-13T14:26:54.950 INFO:tasks.rados.rados.0.smithi026.stdout:2337: rmattr oid 44 current snap is 220 2024-09-13T14:26:54.951 INFO:tasks.rados.rados.0.smithi026.stdout:2337: done (1 left) 2024-09-13T14:26:54.951 INFO:tasks.rados.rados.0.smithi026.stdout:2338: copy_from oid 6 from oid 25 current snap is 220 2024-09-13T14:26:54.951 INFO:tasks.rados.rados.0.smithi026.stdout:2339: snap_create 2024-09-13T14:26:54.958 INFO:tasks.rados.rados.0.smithi026.stdout:2338: finishing copy_from to smithi02683976-6 2024-09-13T14:26:54.958 INFO:tasks.rados.rados.0.smithi026.stdout:2338: got expected ENOENT (src dne) 2024-09-13T14:26:54.958 INFO:tasks.rados.rados.0.smithi026.stdout:2338: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:26:54.972 INFO:tasks.rados.rados.0.smithi026.stdout:2336: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:26:54.974 INFO:tasks.rados.rados.0.smithi026.stdout:2336: finishing copy_from to smithi02683976-32 2024-09-13T14:26:54.974 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 987 (ObjNum 700 snap 200 seq_num 700) dirty exists 2024-09-13T14:26:55.951 INFO:tasks.rados.rados.0.smithi026.stdout:2336: done (2 left) 2024-09-13T14:26:55.952 INFO:tasks.rados.rados.0.smithi026.stdout:2338: done (1 left) 2024-09-13T14:26:55.952 INFO:tasks.rados.rados.0.smithi026.stdout:2339: done (0 left) 2024-09-13T14:26:55.952 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 221 2024-09-13T14:26:55.952 INFO:tasks.rados.rados.0.smithi026.stdout:2340: seq_num 740 ranges {4956160=401408} 2024-09-13T14:26:55.981 INFO:tasks.rados.rados.0.smithi026.stdout:2340: writing smithi02683976-40 from 4956160 to 5357568 tid 1 2024-09-13T14:26:55.981 INFO:tasks.rados.rados.0.smithi026.stdout:2341: setattr oid 23 current snap is 221 2024-09-13T14:26:55.982 INFO:tasks.rados.rados.0.smithi026.stdout:2342: delete oid 21 current snap is 221 2024-09-13T14:26:55.984 INFO:tasks.rados.rados.0.smithi026.stdout:2340: finishing write tid 1 to smithi02683976-40 2024-09-13T14:26:55.984 INFO:tasks.rados.rados.0.smithi026.stdout:2340: finishing write tid 2 to smithi02683976-40 2024-09-13T14:26:55.986 INFO:tasks.rados.rados.0.smithi026.stdout:2340: finishing write tid 3 to smithi02683976-40 2024-09-13T14:26:55.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 957 (ObjNum 740 snap 221 seq_num 740) dirty exists 2024-09-13T14:26:55.986 INFO:tasks.rados.rados.0.smithi026.stdout:2340: left oid 40 (ObjNum 740 snap 221 seq_num 740) 2024-09-13T14:26:55.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1072 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:26:55.989 INFO:tasks.rados.rados.0.smithi026.stdout:2340: done (2 left) 2024-09-13T14:26:55.989 INFO:tasks.rados.rados.0.smithi026.stdout:2341: done (1 left) 2024-09-13T14:26:55.989 INFO:tasks.rados.rados.0.smithi026.stdout:2342: done (0 left) 2024-09-13T14:26:55.989 INFO:tasks.rados.rados.0.smithi026.stdout:2343: read oid 41 snap -1 2024-09-13T14:26:55.990 INFO:tasks.rados.rados.0.smithi026.stdout:2343: expect deleted 2024-09-13T14:26:55.990 INFO:tasks.rados.rados.0.smithi026.stdout:2344: rollback oid 4 current snap is 221 2024-09-13T14:26:55.990 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 218 2024-09-13T14:26:55.990 INFO:tasks.rados.rados.0.smithi026.stdout:2345: setattr oid 3 current snap is 221 2024-09-13T14:26:55.990 INFO:tasks.rados.rados.0.smithi026.stdout:2344: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:26:55.991 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 0 (ObjNum 65 snap 0 seq_num 1315267172) dirty dne 2024-09-13T14:26:55.991 INFO:tasks.rados.rados.0.smithi026.stdout:2343: done (2 left) 2024-09-13T14:26:55.991 INFO:tasks.rados.rados.0.smithi026.stdout:2344: done (1 left) 2024-09-13T14:26:55.991 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 221 2024-09-13T14:26:55.991 INFO:tasks.rados.rados.0.smithi026.stdout:2346: seq_num 742 ranges {5201920=434176} 2024-09-13T14:26:56.021 INFO:tasks.rados.rados.0.smithi026.stdout:2346: writing smithi02683976-31 from 5201920 to 5636096 tid 1 2024-09-13T14:26:56.022 INFO:tasks.rados.rados.0.smithi026.stdout:2347: copy_from oid 4 from oid 9 current snap is 221 2024-09-13T14:26:56.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1224 (ObjNum 33 snap 0 seq_num 519224528) dirty exists 2024-09-13T14:26:56.022 INFO:tasks.rados.rados.0.smithi026.stdout:2345: done (2 left) 2024-09-13T14:26:56.022 INFO:tasks.rados.rados.0.smithi026.stdout:2348: snap_remove snap 215 2024-09-13T14:26:56.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:55 smithi026 ceph-mon[97022]: pgmap v374: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 158 KiB/s wr, 2 op/s; 940 KiB/s, 0 objects/s recovering 2024-09-13T14:26:56.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:55 smithi026 ceph-mon[97022]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:26:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:55 smithi026 ceph-mon[99461]: pgmap v374: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 158 KiB/s wr, 2 op/s; 940 KiB/s, 0 objects/s recovering 2024-09-13T14:26:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:55 smithi026 ceph-mon[99461]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:26:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:55 smithi145 ceph-mon[89145]: pgmap v374: 121 pgs: 121 active+clean; 190 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 158 KiB/s wr, 2 op/s; 940 KiB/s, 0 objects/s recovering 2024-09-13T14:26:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:55 smithi145 ceph-mon[89145]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:26:56.960 INFO:tasks.rados.rados.0.smithi026.stdout:2348: done (2 left) 2024-09-13T14:26:56.960 INFO:tasks.rados.rados.0.smithi026.stdout:2349: snap_create 2024-09-13T14:26:56.960 INFO:tasks.rados.rados.0.smithi026.stdout:2346: finishing write tid 1 to smithi02683976-31 2024-09-13T14:26:56.960 INFO:tasks.rados.rados.0.smithi026.stdout:2346: finishing write tid 2 to smithi02683976-31 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:2346: finishing write tid 3 to smithi02683976-31 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1152 (ObjNum 742 snap 221 seq_num 742) dirty exists 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:2346: left oid 31 (ObjNum 742 snap 221 seq_num 742) 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:2347: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:2347: finishing copy_from to smithi02683976-4 2024-09-13T14:26:56.961 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1025 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:26:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[97022]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:26:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[99461]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:26:57.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:57.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:56 smithi145 ceph-mon[89145]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:26:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:26:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:2346: done (2 left) 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:2347: done (1 left) 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:2349: done (0 left) 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:2350: rollback oid 29 current snap is 222 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 218 2024-09-13T14:26:57.962 INFO:tasks.rados.rados.0.smithi026.stdout:2351: snap_remove snap 219 2024-09-13T14:26:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[99461]: pgmap v377: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:26:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[99461]: osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:26:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[99461]: osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:26:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[97022]: pgmap v377: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:26:58.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[97022]: osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:26:58.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:57 smithi026 ceph-mon[97022]: osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:26:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:57 smithi145 ceph-mon[89145]: pgmap v377: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:26:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:57 smithi145 ceph-mon[89145]: osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:26:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:57 smithi145 ceph-mon[89145]: osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:26:58.950 INFO:tasks.rados.rados.0.smithi026.stdout:2351: done (1 left) 2024-09-13T14:26:58.950 INFO:tasks.rados.rados.0.smithi026.stdout:2352: rollback oid 42 current snap is 222 2024-09-13T14:26:58.950 INFO:tasks.rados.rados.0.smithi026.stdout:2350: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:26:58.950 INFO:tasks.rados.rados.0.smithi026.stdout:2350: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:26:58.951 INFO:tasks.rados.rados.0.smithi026.stdout:2350: finishing rollback tid 2 to smithi02683976-29 2024-09-13T14:26:58.951 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1107 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:26:58.951 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 211 2024-09-13T14:26:58.951 INFO:tasks.rados.rados.0.smithi026.stdout:2350: done (1 left) 2024-09-13T14:26:58.951 INFO:tasks.rados.rados.0.smithi026.stdout:2353: delete oid 11 current snap is 222 2024-09-13T14:26:58.956 INFO:tasks.rados.rados.0.smithi026.stdout:2352: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:26:58.957 INFO:tasks.rados.rados.0.smithi026.stdout:2353: done (1 left) 2024-09-13T14:26:58.957 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 222 2024-09-13T14:26:58.957 INFO:tasks.rados.rados.0.smithi026.stdout:2354: seq_num 744 ranges {7790592=466944} 2024-09-13T14:26:59.003 INFO:tasks.rados.rados.0.smithi026.stdout:2354: writing smithi02683976-49 from 7790592 to 8257536 tid 1 2024-09-13T14:26:59.003 INFO:tasks.rados.rados.0.smithi026.stdout:2355: rmattr oid 12 current snap is 222 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2355: done (2 left) 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2356: rmattr oid 10 current snap is 222 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2352: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2357: copy_from oid 32 from oid 16 current snap is 222 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2352: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1024 (ObjNum 655 snap 184 seq_num 655) dirty exists 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2352: done (3 left) 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2358: copy_from oid 12 from oid 50 current snap is 222 2024-09-13T14:26:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:2359: snap_remove snap 218 2024-09-13T14:26:59.973 INFO:tasks.rados.rados.0.smithi026.stdout:2359: done (4 left) 2024-09-13T14:26:59.973 INFO:tasks.rados.rados.0.smithi026.stdout:2360: rollback oid 1 current snap is 222 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 221 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:2358: finishing copy_from to smithi02683976-12 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:2358: got expected ENOENT (src dne) 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:2358: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:2354: finishing write tid 1 to smithi02683976-49 2024-09-13T14:26:59.974 INFO:tasks.rados.rados.0.smithi026.stdout:2354: finishing write tid 2 to smithi02683976-49 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 960 (ObjNum 694 snap 196 seq_num 694) dirty exists 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:2354: finishing write tid 3 to smithi02683976-49 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 959 (ObjNum 744 snap 222 seq_num 744) dirty exists 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:2354: left oid 49 (ObjNum 744 snap 222 seq_num 744) 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:2357: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:2357: finishing copy_from to smithi02683976-32 2024-09-13T14:26:59.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1101 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2354: done (4 left) 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2356: done (3 left) 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2357: done (2 left) 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2358: done (1 left) 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2361: rollback oid 9 current snap is 222 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 220 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2362: read oid 48 snap -1 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2362: expect deleted 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2363: rollback oid 47 current snap is 222 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 221 2024-09-13T14:26:59.976 INFO:tasks.rados.rados.0.smithi026.stdout:2364: snap_remove snap 211 2024-09-13T14:27:00.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:59 smithi145 ceph-mon[89145]: pgmap v380: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:27:00.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:26:59 smithi145 ceph-mon[89145]: osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:27:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:59 smithi026 ceph-mon[97022]: pgmap v380: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:27:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:26:59 smithi026 ceph-mon[97022]: osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:27:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:59 smithi026 ceph-mon[99461]: pgmap v380: 121 pgs: 121 active+clean; 181 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 5 op/s 2024-09-13T14:27:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:26:59 smithi026 ceph-mon[99461]: osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:27:00.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2364: done (4 left) 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2363: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2360: finishing rollback tid 0 to smithi02683976-1 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2363: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2360: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2363: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 762 (ObjNum 693 snap 195 seq_num 693) dirty exists 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2361: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:2360: finishing rollback tid 2 to smithi02683976-1 2024-09-13T14:27:00.978 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1234 (ObjNum 730 snap 214 seq_num 730) dirty exists 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2361: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2361: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1167 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2365: read oid 12 snap 220 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2365: expect deleted 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2360: done (4 left) 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2361: done (3 left) 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2362: done (2 left) 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2363: done (1 left) 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2366: rollback oid 21 current snap is 222 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 220 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2367: rmattr oid 24 current snap is 222 2024-09-13T14:27:00.979 INFO:tasks.rados.rados.0.smithi026.stdout:2368: snap_create 2024-09-13T14:27:00.985 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 990 (ObjNum 647 snap 182 seq_num 647) dirty exists 2024-09-13T14:27:00.990 INFO:tasks.rados.rados.0.smithi026.stdout:2366: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:27:00.991 INFO:tasks.rados.rados.0.smithi026.stdout:2366: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:27:00.991 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1173 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:01.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:00 smithi026 ceph-mon[97022]: osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:27:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:00 smithi026 ceph-mon[99461]: osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:27:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:00 smithi145 ceph-mon[89145]: osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:27:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:01] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.51.0" 2024-09-13T14:27:01.982 INFO:tasks.rados.rados.0.smithi026.stdout:2365: done (3 left) 2024-09-13T14:27:01.982 INFO:tasks.rados.rados.0.smithi026.stdout:2366: done (2 left) 2024-09-13T14:27:01.982 INFO:tasks.rados.rados.0.smithi026.stdout:2367: done (1 left) 2024-09-13T14:27:01.983 INFO:tasks.rados.rados.0.smithi026.stdout:2368: done (0 left) 2024-09-13T14:27:01.983 INFO:tasks.rados.rados.0.smithi026.stdout:2369: read oid 21 snap -1 2024-09-13T14:27:01.983 INFO:tasks.rados.rados.0.smithi026.stdout:2369: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:27:01.983 INFO:tasks.rados.rados.0.smithi026.stdout:2370: delete oid 10 current snap is 223 2024-09-13T14:27:01.988 INFO:tasks.rados.rados.0.smithi026.stdout:2370: done (1 left) 2024-09-13T14:27:01.988 INFO:tasks.rados.rados.0.smithi026.stdout:2371: rollback oid 28 current snap is 223 2024-09-13T14:27:01.988 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 220 2024-09-13T14:27:01.989 INFO:tasks.rados.rados.0.smithi026.stdout:2372: snap_remove snap 222 2024-09-13T14:27:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[97022]: pgmap v383: 121 pgs: 121 active+clean; 189 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-13T14:27:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[97022]: osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:27:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[99461]: pgmap v383: 121 pgs: 121 active+clean; 189 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-13T14:27:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[99461]: osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:27:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:01 smithi145 ceph-mon[89145]: pgmap v383: 121 pgs: 121 active+clean; 189 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-13T14:27:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:01 smithi145 ceph-mon[89145]: osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:27:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:02.988 INFO:tasks.rados.rados.0.smithi026.stdout:2372: done (2 left) 2024-09-13T14:27:02.988 INFO:tasks.rados.rados.0.smithi026.stdout:2373: snap_create 2024-09-13T14:27:02.988 INFO:tasks.rados.rados.0.smithi026.stdout:2371: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:27:02.988 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:27:02.988 INFO:tasks.rados.rados.0.smithi026.stdout:2369: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:27:03.323 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:02 smithi145 ceph-mon[89145]: osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:27:03.324 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:27:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:02 smithi026 ceph-mon[99461]: osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:27:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:27:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:02 smithi026 ceph-mon[97022]: osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:27:03.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T14:27:03.950 INFO:tasks.rados.rados.0.smithi026.stdout:2369: done (2 left) 2024-09-13T14:27:03.950 INFO:tasks.rados.rados.0.smithi026.stdout:2371: done (1 left) 2024-09-13T14:27:03.951 INFO:tasks.rados.rados.0.smithi026.stdout:2373: done (0 left) 2024-09-13T14:27:03.951 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 224 2024-09-13T14:27:03.951 INFO:tasks.rados.rados.0.smithi026.stdout:2374: seq_num 746 ranges {5357568=499712} 2024-09-13T14:27:03.982 INFO:tasks.rados.rados.0.smithi026.stdout:2374: writing smithi02683976-40 from 5357568 to 5857280 tid 1 2024-09-13T14:27:03.982 INFO:tasks.rados.rados.0.smithi026.stdout:2375: setattr oid 18 current snap is 224 2024-09-13T14:27:03.983 INFO:tasks.rados.rados.0.smithi026.stdout:2376: setattr oid 7 current snap is 224 2024-09-13T14:27:03.984 INFO:tasks.rados.rados.0.smithi026.stdout:2377: rollback oid 12 current snap is 224 2024-09-13T14:27:03.984 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 220 2024-09-13T14:27:03.984 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 224 2024-09-13T14:27:03.984 INFO:tasks.rados.rados.0.smithi026.stdout:2378: seq_num 747 ranges {3850240=516096} 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2378: writing smithi02683976-4 from 3850240 to 4366336 tid 1 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2379: read oid 32 snap 223 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2379: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2380: rmattr oid 2 current snap is 224 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2377: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-13T14:27:04.010 INFO:tasks.rados.rados.0.smithi026.stdout:2377: done (6 left) 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2380: done (5 left) 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2381: delete oid 25 current snap is 224 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2374: finishing write tid 1 to smithi02683976-40 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1122 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1102 (ObjNum 65 snap 0 seq_num 1381785439) dirty exists 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2374: finishing write tid 2 to smithi02683976-40 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2374: finishing write tid 3 to smithi02683976-40 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 963 (ObjNum 746 snap 224 seq_num 746) dirty exists 2024-09-13T14:27:04.011 INFO:tasks.rados.rados.0.smithi026.stdout:2374: left oid 40 (ObjNum 746 snap 224 seq_num 746) 2024-09-13T14:27:04.016 INFO:tasks.rados.rados.0.smithi026.stdout:2378: finishing write tid 1 to smithi02683976-4 2024-09-13T14:27:04.016 INFO:tasks.rados.rados.0.smithi026.stdout:2378: finishing write tid 2 to smithi02683976-4 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:2374: done (5 left) 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:2375: done (4 left) 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:2376: done (3 left) 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:2381: done (2 left) 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 224 2024-09-13T14:27:04.018 INFO:tasks.rados.rados.0.smithi026.stdout:2382: seq_num 749 ranges {4497408=548864} 2024-09-13T14:27:04.046 INFO:tasks.rados.rados.0.smithi026.stdout:2382: writing smithi02683976-7 from 4497408 to 5046272 tid 1 2024-09-13T14:27:04.047 INFO:tasks.rados.rados.0.smithi026.stdout:2383: delete oid 25 current snap is 224 2024-09-13T14:27:04.047 INFO:tasks.rados.rados.0.smithi026.stdout:2378: finishing write tid 3 to smithi02683976-4 2024-09-13T14:27:04.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1029 (ObjNum 747 snap 224 seq_num 747) dirty exists 2024-09-13T14:27:04.047 INFO:tasks.rados.rados.0.smithi026.stdout:2378: left oid 4 (ObjNum 747 snap 224 seq_num 747) 2024-09-13T14:27:04.047 INFO:tasks.rados.rados.0.smithi026.stdout:2379: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:04.390 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:03 smithi145 ceph-mon[89145]: pgmap v386: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:27:04.390 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:03 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:27:04.390 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:03 smithi145 ceph-mon[89145]: Upgrade: osd.6 is safe to restart 2024-09-13T14:27:04.390 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:03 smithi145 ceph-mon[89145]: osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:27:04.390 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:03 smithi145 ceph-mon[89145]: osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:27:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[97022]: pgmap v386: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[97022]: Upgrade: osd.6 is safe to restart 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[97022]: osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[97022]: osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[99461]: pgmap v386: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[99461]: Upgrade: osd.6 is safe to restart 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[99461]: osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:27:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:03 smithi026 ceph-mon[99461]: osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2382: finishing write tid 1 to smithi02683976-7 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2382: finishing write tid 2 to smithi02683976-7 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2382: finishing write tid 3 to smithi02683976-7 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1124 (ObjNum 749 snap 224 seq_num 749) dirty exists 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2382: left oid 7 (ObjNum 749 snap 224 seq_num 749) 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2378: done (3 left) 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2379: done (2 left) 2024-09-13T14:27:04.487 INFO:tasks.rados.rados.0.smithi026.stdout:2382: done (1 left) 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2383: done (0 left) 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2384: rmattr oid 23 current snap is 224 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2385: read oid 49 snap 221 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2385: expect (ObjNum 709 snap 208 seq_num 709) 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2386: rollback oid 18 current snap is 224 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 223 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2387: read oid 26 snap -1 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2387: expect (ObjNum 735 snap 218 seq_num 735) 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2388: read oid 48 snap -1 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2388: expect deleted 2024-09-13T14:27:04.488 INFO:tasks.rados.rados.0.smithi026.stdout:2389: delete oid 20 current snap is 224 2024-09-13T14:27:04.490 INFO:tasks.rados.rados.0.smithi026.stdout:2388: done (5 left) 2024-09-13T14:27:04.490 INFO:tasks.rados.rados.0.smithi026.stdout:2389: done (4 left) 2024-09-13T14:27:04.490 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 224 2024-09-13T14:27:04.490 INFO:tasks.rados.rados.0.smithi026.stdout:2390: seq_num 752 ranges {0=598016} 2024-09-13T14:27:04.494 INFO:tasks.rados.rados.0.smithi026.stdout:2390: writing smithi02683976-15 from 0 to 598016 tid 1 2024-09-13T14:27:04.494 INFO:tasks.rados.rados.0.smithi026.stdout:2391: snap_create 2024-09-13T14:27:04.494 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1103 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:27:04.494 INFO:tasks.rados.rados.0.smithi026.stdout:2386: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:27:04.495 INFO:tasks.rados.rados.0.smithi026.stdout:2386: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:27:04.495 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1105 (ObjNum 65 snap 0 seq_num 1381785439) dirty dne 2024-09-13T14:27:04.495 INFO:tasks.rados.rados.0.smithi026.stdout:2387: expect (ObjNum 735 snap 218 seq_num 735) 2024-09-13T14:27:04.532 INFO:tasks.rados.rados.0.smithi026.stdout:2390: finishing write tid 1 to smithi02683976-15 2024-09-13T14:27:04.532 INFO:tasks.rados.rados.0.smithi026.stdout:2390: finishing write tid 2 to smithi02683976-15 2024-09-13T14:27:04.533 INFO:tasks.rados.rados.0.smithi026.stdout:2390: finishing write tid 3 to smithi02683976-15 2024-09-13T14:27:04.533 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1031 (ObjNum 752 snap 224 seq_num 752) dirty exists 2024-09-13T14:27:04.536 INFO:tasks.rados.rados.0.smithi026.stdout:2390: left oid 15 (ObjNum 752 snap 224 seq_num 752) 2024-09-13T14:27:04.536 INFO:tasks.rados.rados.0.smithi026.stdout:2385: expect (ObjNum 709 snap 208 seq_num 709) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2384: done (5 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2385: done (4 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2386: done (3 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2387: done (2 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2390: done (1 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2391: done (0 left) 2024-09-13T14:27:05.428 INFO:tasks.rados.rados.0.smithi026.stdout:2392: setattr oid 39 current snap is 225 2024-09-13T14:27:05.429 INFO:tasks.rados.rados.0.smithi026.stdout:2393: rmattr oid 7 current snap is 225 2024-09-13T14:27:05.429 INFO:tasks.rados.rados.0.smithi026.stdout:2394: read oid 26 snap -1 2024-09-13T14:27:05.429 INFO:tasks.rados.rados.0.smithi026.stdout:2394: expect (ObjNum 735 snap 218 seq_num 735) 2024-09-13T14:27:05.429 INFO:tasks.rados.rados.0.smithi026.stdout:2395: rollback oid 35 current snap is 225 2024-09-13T14:27:05.430 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 35 to 223 2024-09-13T14:27:05.430 INFO:tasks.rados.rados.0.smithi026.stdout:2396: rollback oid 43 current snap is 225 2024-09-13T14:27:05.430 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 43 to 224 2024-09-13T14:27:05.430 INFO:tasks.rados.rados.0.smithi026.stdout:2397: snap_remove snap 220 2024-09-13T14:27:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:05 smithi145 ceph-mon[89145]: Upgrade: Updating osd.6 2024-09-13T14:27:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:27:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:05 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:05.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[99461]: Upgrade: Updating osd.6 2024-09-13T14:27:05.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[97022]: Upgrade: Updating osd.6 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:27:05.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:05 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:06.433 INFO:tasks.rados.rados.0.smithi026.stdout:2397: done (5 left) 2024-09-13T14:27:06.433 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 225 2024-09-13T14:27:06.433 INFO:tasks.rados.rados.0.smithi026.stdout:2398: seq_num 753 ranges {0=614400} 2024-09-13T14:27:06.437 INFO:tasks.rados.rados.0.smithi026.stdout:2398: writing smithi02683976-48 from 0 to 614400 tid 1 2024-09-13T14:27:06.437 INFO:tasks.rados.rados.0.smithi026.stdout:2399: rollback oid 9 current snap is 225 2024-09-13T14:27:06.437 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 223 2024-09-13T14:27:06.437 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 225 2024-09-13T14:27:06.437 INFO:tasks.rados.rados.0.smithi026.stdout:2400: seq_num 754 ranges {0=630784} 2024-09-13T14:27:06.441 INFO:tasks.rados.rados.0.smithi026.stdout:2400: writing smithi02683976-17 from 0 to 630784 tid 1 2024-09-13T14:27:06.442 INFO:tasks.rados.rados.0.smithi026.stdout:2401: snap_create 2024-09-13T14:27:06.442 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1125 (ObjNum 749 snap 224 seq_num 749) dirty exists 2024-09-13T14:27:06.442 INFO:tasks.rados.rados.0.smithi026.stdout:2394: expect (ObjNum 735 snap 218 seq_num 735) 2024-09-13T14:27:06.481 INFO:tasks.rados.rados.0.smithi026.stdout:2395: finishing rollback tid 0 to smithi02683976-35 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2396: finishing rollback tid 0 to smithi02683976-43 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 991 (ObjNum 662 snap 188 seq_num 662) dirty exists 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2396: finishing rollback tid 1 to smithi02683976-43 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2396: finishing rollback tid 2 to smithi02683976-43 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1123 (ObjNum 660 snap 188 seq_num 660) dirty exists 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2395: finishing rollback tid 1 to smithi02683976-35 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2395: finishing rollback tid 2 to smithi02683976-35 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1248 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2399: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2399: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2398: finishing write tid 1 to smithi02683976-48 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2398: finishing write tid 2 to smithi02683976-48 2024-09-13T14:27:06.482 INFO:tasks.rados.rados.0.smithi026.stdout:2400: finishing write tid 1 to smithi02683976-17 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2400: finishing write tid 2 to smithi02683976-17 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2398: finishing write tid 3 to smithi02683976-48 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 764 (ObjNum 753 snap 225 seq_num 753) dirty exists 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2398: left oid 48 (ObjNum 753 snap 225 seq_num 753) 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2399: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1185 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2400: finishing write tid 3 to smithi02683976-17 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 832 (ObjNum 754 snap 225 seq_num 754) dirty exists 2024-09-13T14:27:06.483 INFO:tasks.rados.rados.0.smithi026.stdout:2400: left oid 17 (ObjNum 754 snap 225 seq_num 754) 2024-09-13T14:27:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:06 smithi145 ceph-mon[89145]: Deploying daemon osd.6 on smithi145 2024-09-13T14:27:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:06 smithi145 ceph-mon[89145]: pgmap v389: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-13T14:27:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:06 smithi145 ceph-mon[89145]: osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:27:06.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[97022]: Deploying daemon osd.6 on smithi145 2024-09-13T14:27:06.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[97022]: pgmap v389: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-13T14:27:06.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[97022]: osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:27:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[99461]: Deploying daemon osd.6 on smithi145 2024-09-13T14:27:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[99461]: pgmap v389: 121 pgs: 121 active+clean; 179 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-13T14:27:06.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:06 smithi026 ceph-mon[99461]: osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2392: done (8 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2393: done (7 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2394: done (6 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2395: done (5 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2396: done (4 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2398: done (3 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2399: done (2 left) 2024-09-13T14:27:07.439 INFO:tasks.rados.rados.0.smithi026.stdout:2400: done (1 left) 2024-09-13T14:27:07.440 INFO:tasks.rados.rados.0.smithi026.stdout:2401: done (0 left) 2024-09-13T14:27:07.440 INFO:tasks.rados.rados.0.smithi026.stdout:2402: snap_remove snap 223 2024-09-13T14:27:07.611 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:07 smithi145 ceph-mon[89145]: osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:27:07.611 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:07 smithi145 ceph-mon[89145]: pgmap v392: 121 pgs: 121 active+clean; 162 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 394 KiB/s wr, 18 op/s 2024-09-13T14:27:07.612 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:07 smithi145 systemd[1]: Stopping Ceph osd.6 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:27:07.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:07 smithi026 ceph-mon[99461]: osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:27:07.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:07 smithi026 ceph-mon[99461]: pgmap v392: 121 pgs: 121 active+clean; 162 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 394 KiB/s wr, 18 op/s 2024-09-13T14:27:07.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:07 smithi026 ceph-mon[97022]: osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:27:07.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:07 smithi026 ceph-mon[97022]: pgmap v392: 121 pgs: 121 active+clean; 162 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 394 KiB/s wr, 18 op/s 2024-09-13T14:27:07.955 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[48379]: 2024-09-13T14:27:07.609+0000 7fcee71ce640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:27:07.955 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[48379]: 2024-09-13T14:27:07.609+0000 7fcee71ce640 -1 osd.6 529 *** Got signal Terminated *** 2024-09-13T14:27:07.955 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[48379]: 2024-09-13T14:27:07.609+0000 7fcee71ce640 -1 osd.6 529 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:2402: done (0 left) 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:2403: read oid 16 snap -1 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:2403: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:2404: copy_from oid 39 from oid 4 current snap is 226 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 226 2024-09-13T14:27:08.447 INFO:tasks.rados.rados.0.smithi026.stdout:2405: seq_num 755 ranges {3850240=647168} 2024-09-13T14:27:08.474 INFO:tasks.rados.rados.0.smithi026.stdout:2405: writing smithi02683976-27 from 3850240 to 4497408 tid 1 2024-09-13T14:27:08.474 INFO:tasks.rados.rados.0.smithi026.stdout:2406: copy_from oid 26 from oid 43 current snap is 226 2024-09-13T14:27:08.474 INFO:tasks.rados.rados.0.smithi026.stdout:2407: rollback oid 2 current snap is 226 2024-09-13T14:27:08.474 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 225 2024-09-13T14:27:08.474 INFO:tasks.rados.rados.0.smithi026.stdout:2408: rollback oid 35 current snap is 226 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 35 to 225 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:2409: rollback oid 22 current snap is 226 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 225 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:2410: copy_from oid 49 from oid 33 current snap is 226 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:2411: rollback oid 42 current snap is 226 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 225 2024-09-13T14:27:08.475 INFO:tasks.rados.rados.0.smithi026.stdout:2412: delete oid 47 current snap is 226 2024-09-13T14:27:08.478 INFO:tasks.rados.rados.0.smithi026.stdout:2412: done (9 left) 2024-09-13T14:27:08.478 INFO:tasks.rados.rados.0.smithi026.stdout:2413: read oid 5 snap 224 2024-09-13T14:27:08.478 INFO:tasks.rados.rados.0.smithi026.stdout:2413: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:27:08.478 INFO:tasks.rados.rados.0.smithi026.stdout:2414: snap_remove snap 221 2024-09-13T14:27:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:08 smithi145 ceph-mon[89145]: osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:27:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:08 smithi145 ceph-mon[89145]: osd.6 marked itself down and dead 2024-09-13T14:27:08.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:08 smithi026 ceph-mon[99461]: osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:27:08.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:08 smithi026 ceph-mon[99461]: osd.6 marked itself down and dead 2024-09-13T14:27:08.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:08 smithi026 ceph-mon[97022]: osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:27:08.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:08 smithi026 ceph-mon[97022]: osd.6 marked itself down and dead 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2414: done (10 left) 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2415: snap_create 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2411: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2411: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2411: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1027 (ObjNum 655 snap 184 seq_num 655) dirty exists 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2407: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:27:08.955 INFO:tasks.rados.rados.0.smithi026.stdout:2407: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2407: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 994 (ObjNum 700 snap 200 seq_num 700) dirty exists 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2405: finishing write tid 1 to smithi02683976-27 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2405: finishing write tid 2 to smithi02683976-27 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2408: finishing rollback tid 0 to smithi02683976-35 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2408: finishing rollback tid 1 to smithi02683976-35 2024-09-13T14:27:08.956 INFO:tasks.rados.rados.0.smithi026.stdout:2403: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:09.259 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:08 smithi145 podman[104825]: 2024-09-13 14:27:08.995141495 +0000 UTC m=+1.480628224 container died 640c9cc2fdf49f24d39529fdb20eac1ec19d55dbdd37c9bd25fc2829f153434e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:27:09.404 INFO:tasks.rados.rados.0.smithi026.stdout:2405: finishing write tid 3 to smithi02683976-27 2024-09-13T14:27:09.404 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1257 (ObjNum 755 snap 226 seq_num 755) dirty exists 2024-09-13T14:27:09.404 INFO:tasks.rados.rados.0.smithi026.stdout:2405: left oid 27 (ObjNum 755 snap 226 seq_num 755) 2024-09-13T14:27:09.405 INFO:tasks.rados.rados.0.smithi026.stdout:2408: finishing rollback tid 2 to smithi02683976-35 2024-09-13T14:27:09.405 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1261 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:09.405 INFO:tasks.rados.rados.0.smithi026.stdout:2413: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:27:09.677 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:09 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:09.677 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:09 smithi145 ceph-mon[89145]: osdmap e530: 8 total, 7 up, 8 in 2024-09-13T14:27:09.677 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:09 smithi145 ceph-mon[89145]: pgmap v395: 121 pgs: 14 stale+active+clean, 107 active+clean; 163 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 528 KiB/s wr, 31 op/s 2024-09-13T14:27:09.677 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:09 smithi145 ceph-mon[89145]: osdmap e531: 8 total, 7 up, 8 in 2024-09-13T14:27:09.677 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:09 smithi145 podman[104825]: 2024-09-13 14:27:09.37481124 +0000 UTC m=+1.860297962 container cleanup 640c9cc2fdf49f24d39529fdb20eac1ec19d55dbdd37c9bd25fc2829f153434e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:27:09.677 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:09 smithi145 bash[104825]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6 2024-09-13T14:27:09.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:09.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[97022]: osdmap e530: 8 total, 7 up, 8 in 2024-09-13T14:27:09.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[97022]: pgmap v395: 121 pgs: 14 stale+active+clean, 107 active+clean; 163 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 528 KiB/s wr, 31 op/s 2024-09-13T14:27:09.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[97022]: osdmap e531: 8 total, 7 up, 8 in 2024-09-13T14:27:09.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:09.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[99461]: osdmap e530: 8 total, 7 up, 8 in 2024-09-13T14:27:09.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[99461]: pgmap v395: 121 pgs: 14 stale+active+clean, 107 active+clean; 163 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 528 KiB/s wr, 31 op/s 2024-09-13T14:27:09.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:09 smithi026 ceph-mon[99461]: osdmap e531: 8 total, 7 up, 8 in 2024-09-13T14:27:09.955 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:09 smithi145 podman[104856]: 2024-09-13 14:27:09.676449554 +0000 UTC m=+0.683270741 container remove 640c9cc2fdf49f24d39529fdb20eac1ec19d55dbdd37c9bd25fc2829f153434e (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:2410: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:2410: finishing copy_from to smithi02683976-49 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1150 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:2409: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:2409: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:2409: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:27:10.185 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 976 (ObjNum 691 snap 195 seq_num 691) dirty exists 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2406: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2403: done (10 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2405: done (9 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2407: done (8 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2408: done (7 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2409: done (6 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2410: done (5 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2411: done (4 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2413: done (3 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2415: done (2 left) 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2416: delete oid 3 current snap is 227 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2406: finishing copy_from to smithi02683976-26 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1123 (ObjNum 660 snap 188 seq_num 660) dirty exists 2024-09-13T14:27:10.186 INFO:tasks.rados.rados.0.smithi026.stdout:2404: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:2404: finishing copy_from to smithi02683976-39 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1029 (ObjNum 747 snap 224 seq_num 747) dirty exists 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:2404: done (2 left) 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:2406: done (1 left) 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:2416: done (0 left) 2024-09-13T14:27:10.187 INFO:tasks.rados.rados.0.smithi026.stdout:2417: snap_create 2024-09-13T14:27:10.288 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:10.288 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:09 smithi145 podman[104948]: 2024-09-13 14:27:09.888998737 +0000 UTC m=+0.015840079 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:10.288 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:09 smithi145 podman[104948]: 2024-09-13 14:27:09.994081465 +0000 UTC m=+0.120922807 container create 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0) 2024-09-13T14:27:10.289 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:10 smithi145 podman[104948]: 2024-09-13 14:27:10.196592263 +0000 UTC m=+0.323433602 container init 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:27:10.289 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:10 smithi145 podman[104948]: 2024-09-13 14:27:10.20162937 +0000 UTC m=+0.328470709 container start 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:27:10.289 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:10 smithi145 podman[104948]: 2024-09-13 14:27:10.287858089 +0000 UTC m=+0.414699429 container attach 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0) 2024-09-13T14:27:10.685 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:10 smithi145 podman[104948]: 2024-09-13 14:27:10.39604063 +0000 UTC m=+0.522881969 container died 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:27:10.959 INFO:tasks.rados.rados.0.smithi026.stdout:2417: done (0 left) 2024-09-13T14:27:10.959 INFO:tasks.rados.rados.0.smithi026.stdout:2418: rollback oid 30 current snap is 228 2024-09-13T14:27:10.959 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 225 2024-09-13T14:27:10.960 INFO:tasks.rados.rados.0.smithi026.stdout:2419: rollback oid 21 current snap is 228 2024-09-13T14:27:10.960 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 225 2024-09-13T14:27:10.960 INFO:tasks.rados.rados.0.smithi026.stdout:2420: delete oid 10 current snap is 228 2024-09-13T14:27:10.964 INFO:tasks.rados.rados.0.smithi026.stdout:2419: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:27:10.964 INFO:tasks.rados.rados.0.smithi026.stdout:2420: done (2 left) 2024-09-13T14:27:10.964 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 228 2024-09-13T14:27:10.964 INFO:tasks.rados.rados.0.smithi026.stdout:2421: seq_num 759 ranges {614400=720896} 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:2421: writing smithi02683976-48 from 614400 to 1335296 tid 1 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:2422: snap_create 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:2419: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:2419: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1195 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:10.972 INFO:tasks.rados.rados.0.smithi026.stdout:2418: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:27:10.973 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-13T14:27:10.976 INFO:tasks.rados.rados.0.smithi026.stdout:2421: finishing write tid 1 to smithi02683976-48 2024-09-13T14:27:10.976 INFO:tasks.rados.rados.0.smithi026.stdout:2421: finishing write tid 2 to smithi02683976-48 2024-09-13T14:27:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:2421: finishing write tid 3 to smithi02683976-48 2024-09-13T14:27:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 767 (ObjNum 759 snap 228 seq_num 759) dirty exists 2024-09-13T14:27:10.977 INFO:tasks.rados.rados.0.smithi026.stdout:2421: left oid 48 (ObjNum 759 snap 228 seq_num 759) 2024-09-13T14:27:11.137 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:27:11.316 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:10 smithi145 ceph-mon[89145]: osdmap e532: 8 total, 7 up, 8 in 2024-09-13T14:27:11.316 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[104993]: 2024-09-13 14:27:11.075229723 +0000 UTC m=+0.669688266 container remove 3f9f1671b05de129172ed2141ada8f9eb7940d8e540aa1832a5b562f94467de4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:27:11.316 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T14:27:11.317 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 systemd[1]: Stopped Ceph osd.6 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:27:11.317 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.6.service: Consumed 12.944s CPU time. 2024-09-13T14:27:11.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:10 smithi026 ceph-mon[99461]: osdmap e532: 8 total, 7 up, 8 in 2024-09-13T14:27:11.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:10 smithi026 ceph-mon[97022]: osdmap e532: 8 total, 7 up, 8 in 2024-09-13T14:27:11.645 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 systemd[1]: Starting Ceph osd.6 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:27:11.645 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[105097]: 2024-09-13 14:27:11.428501865 +0000 UTC m=+0.015794994 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:11.645 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[105097]: 2024-09-13 14:27:11.535122083 +0000 UTC m=+0.122415208 container create b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:27:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:11] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.51.0" 2024-09-13T14:27:11.726 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:27:11.726 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 95s ago 10m 18.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:27:11.726 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (5m) 27s ago 10m 87.1M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (12m) 95s ago 12m 15.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (11m) 27s ago 11m 23.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (4m) 27s ago 15m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (5m) 95s ago 17m 523M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 95s ago 17m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (4m) 27s ago 15m 46.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 95s ago 15m 46.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 95s ago 10m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (5m) 27s ago 10m 10.7M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 95s ago 15m 152M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 95s ago 14m 144M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 95s ago 14m 108M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (99s) 95s ago 13m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:27:11.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (67s) 27s ago 13m 106M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:27:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (32s) 27s ago 13m 12.2M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 b9422108344c 2024-09-13T14:27:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (12m) 27s ago 12m 159M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 640c9cc2fdf4 2024-09-13T14:27:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (12m) 27s ago 12m 169M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:27:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (4m) 27s ago 11m 49.8M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:27:11.913 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[105097]: 2024-09-13 14:27:11.779257899 +0000 UTC m=+0.366551043 container init b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T14:27:11.913 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[105097]: 2024-09-13 14:27:11.784574188 +0000 UTC m=+0.371867314 container start b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T14:27:11.913 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 podman[105097]: 2024-09-13 14:27:11.887580094 +0000 UTC m=+0.474873225 container attach b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T14:27:11.965 INFO:tasks.rados.rados.0.smithi026.stdout:2418: done (3 left) 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2419: done (2 left) 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2421: done (1 left) 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2422: done (0 left) 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2423: rmattr oid 36 current snap is 229 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2423: done (0 left) 2024-09-13T14:27:11.966 INFO:tasks.rados.rados.0.smithi026.stdout:2424: delete oid 24 current snap is 229 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2424: done (0 left) 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2425: read oid 46 snap -1 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2425: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2426: read oid 23 snap -1 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2426: expect (ObjNum 712 snap 208 seq_num 712) 2024-09-13T14:27:11.971 INFO:tasks.rados.rados.0.smithi026.stdout:2427: snap_create 2024-09-13T14:27:12.014 INFO:tasks.rados.rados.0.smithi026.stdout:2426: expect (ObjNum 712 snap 208 seq_num 712) 2024-09-13T14:27:12.084 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:27:12.084 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 6 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:27:12.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:12.086 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:27:12.086 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-13T14:27:12.086 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 11 2024-09-13T14:27:12.086 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:27:12.086 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:27:12.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: pgmap v398: 121 pgs: 19 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 95 active+clean; 168 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.6 MiB/s wr, 19 op/s; 4/243 objects degraded (1.646%) 2024-09-13T14:27:12.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 4/243 objects degraded (1.646%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:27:12.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: osdmap e533: 8 total, 7 up, 8 in 2024-09-13T14:27:12.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: pgmap v398: 121 pgs: 19 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 95 active+clean; 168 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.6 MiB/s wr, 19 op/s; 4/243 objects degraded (1.646%) 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 4/243 objects degraded (1.646%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: osdmap e533: 8 total, 7 up, 8 in 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:12.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: pgmap v398: 121 pgs: 19 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 95 active+clean; 168 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 2.6 MiB/s wr, 19 op/s; 4/243 objects degraded (1.646%) 2024-09-13T14:27:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 4/243 objects degraded (1.646%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:27:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: osdmap e533: 8 total, 7 up, 8 in 2024-09-13T14:27:12.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:12.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:12.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 bash[105097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 bash[105097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 bash[105097]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 bash[105097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:11 smithi145 bash[105097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-13T14:27:12.207 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-13T14:27:12.379 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:27:12.379 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:27:12.379 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:27:12.379 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "11/20 daemons upgraded", 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:27:12.380 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:27:12.650 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T14:27:12.650 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate[105113]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 bash[105097]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-13T14:27:12.651 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:12 smithi145 podman[105097]: 2024-09-13 14:27:12.348064876 +0000 UTC m=+0.935358020 container died b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD) 2024-09-13T14:27:12.804 INFO:tasks.rados.rados.0.smithi026.stdout:2425: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:27:13.264 INFO:tasks.rados.rados.0.smithi026.stdout:2425: done (2 left) 2024-09-13T14:27:13.265 INFO:tasks.rados.rados.0.smithi026.stdout:2426: done (1 left) 2024-09-13T14:27:13.265 INFO:tasks.rados.rados.0.smithi026.stdout:2427: done (0 left) 2024-09-13T14:27:13.265 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 230 2024-09-13T14:27:13.265 INFO:tasks.rados.rados.0.smithi026.stdout:2428: seq_num 761 ranges {4497408=753664} 2024-09-13T14:27:13.266 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 podman[105252]: 2024-09-13 14:27:13.058686728 +0000 UTC m=+0.704353977 container remove b1350e589457163b9e790366cd2345f3cf16ea445a391d5a8789d72a956772ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T14:27:13.266 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-mon[89145]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.266 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-mon[89145]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.266 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-mon[89145]: osdmap e534: 8 total, 7 up, 8 in 2024-09-13T14:27:13.266 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3650272765' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:13.266 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:12 smithi145 ceph-mon[89145]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.293 INFO:tasks.rados.rados.0.smithi026.stdout:2428: writing smithi02683976-13 from 4497408 to 5251072 tid 1 2024-09-13T14:27:13.293 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 230 2024-09-13T14:27:13.294 INFO:tasks.rados.rados.0.smithi026.stdout:2429: seq_num 762 ranges {630784=770048} 2024-09-13T14:27:13.301 INFO:tasks.rados.rados.0.smithi026.stdout:2429: writing smithi02683976-17 from 630784 to 1400832 tid 1 2024-09-13T14:27:13.301 INFO:tasks.rados.rados.0.smithi026.stdout:2430: read oid 30 snap -1 2024-09-13T14:27:13.301 INFO:tasks.rados.rados.0.smithi026.stdout:2430: expect deleted 2024-09-13T14:27:13.301 INFO:tasks.rados.rados.0.smithi026.stdout:2431: snap_remove snap 227 2024-09-13T14:27:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[97022]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[97022]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[97022]: osdmap e534: 8 total, 7 up, 8 in 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3650272765' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[97022]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[99461]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[99461]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[99461]: osdmap e534: 8 total, 7 up, 8 in 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3650272765' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:12 smithi026 ceph-mon[99461]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.529 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 podman[105315]: 2024-09-13 14:27:13.167162386 +0000 UTC m=+0.015819449 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:13.529 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 podman[105315]: 2024-09-13 14:27:13.284692138 +0000 UTC m=+0.133349186 container create 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:27:13.530 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 podman[105315]: 2024-09-13 14:27:13.503465012 +0000 UTC m=+0.352122072 container init 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:27:13.530 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 podman[105315]: 2024-09-13 14:27:13.508281704 +0000 UTC m=+0.356938753 container start 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:27:13.957 INFO:tasks.rados.rados.0.smithi026.stdout:2431: done (3 left) 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2432: delete oid 47 current snap is 230 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2428: finishing write tid 1 to smithi02683976-13 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2428: finishing write tid 2 to smithi02683976-13 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2428: finishing write tid 3 to smithi02683976-13 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1142 (ObjNum 761 snap 230 seq_num 761) dirty exists 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2428: left oid 13 (ObjNum 761 snap 230 seq_num 761) 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2429: finishing write tid 1 to smithi02683976-17 2024-09-13T14:27:13.958 INFO:tasks.rados.rados.0.smithi026.stdout:2429: finishing write tid 2 to smithi02683976-17 2024-09-13T14:27:13.959 INFO:tasks.rados.rados.0.smithi026.stdout:2429: finishing write tid 3 to smithi02683976-17 2024-09-13T14:27:13.959 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 834 (ObjNum 762 snap 230 seq_num 762) dirty exists 2024-09-13T14:27:13.959 INFO:tasks.rados.rados.0.smithi026.stdout:2429: left oid 17 (ObjNum 762 snap 230 seq_num 762) 2024-09-13T14:27:13.959 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-osd[105353]: -- 172.21.15.145:0/2014287249 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558fe8717a40 con 0x558fe86f5c00 2024-09-13T14:27:13.959 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 bash[105315]: 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 2024-09-13T14:27:13.960 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:13 smithi145 systemd[1]: Started Ceph osd.6 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:27:13.960 INFO:tasks.rados.rados.0.smithi026.stdout:2428: done (3 left) 2024-09-13T14:27:13.960 INFO:tasks.rados.rados.0.smithi026.stdout:2429: done (2 left) 2024-09-13T14:27:13.961 INFO:tasks.rados.rados.0.smithi026.stdout:2430: done (1 left) 2024-09-13T14:27:13.961 INFO:tasks.rados.rados.0.smithi026.stdout:2432: done (0 left) 2024-09-13T14:27:13.961 INFO:tasks.rados.rados.0.smithi026.stdout:2433: snap_create 2024-09-13T14:27:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: pgmap v401: 121 pgs: 28 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 84 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 10 op/s; 20/243 objects degraded (8.230%) 2024-09-13T14:27:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: osdmap e535: 8 total, 7 up, 8 in 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[97022]: osdmap e536: 8 total, 7 up, 8 in 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: pgmap v401: 121 pgs: 28 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 84 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 10 op/s; 20/243 objects degraded (8.230%) 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: osdmap e535: 8 total, 7 up, 8 in 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:14.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:13 smithi026 ceph-mon[99461]: osdmap e536: 8 total, 7 up, 8 in 2024-09-13T14:27:14.440 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: pgmap v401: 121 pgs: 28 active+undersized, 1 stale+active+clean, 8 active+undersized+degraded, 84 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.5 MiB/s wr, 10 op/s; 20/243 objects degraded (8.230%) 2024-09-13T14:27:14.440 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: osdmap e535: 8 total, 7 up, 8 in 2024-09-13T14:27:14.441 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.441 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:14.441 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:14.441 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:13 smithi145 ceph-mon[89145]: osdmap e536: 8 total, 7 up, 8 in 2024-09-13T14:27:14.954 INFO:tasks.rados.rados.0.smithi026.stdout:2433: done (0 left) 2024-09-13T14:27:14.954 INFO:tasks.rados.rados.0.smithi026.stdout:2434: delete oid 28 current snap is 231 2024-09-13T14:27:14.958 INFO:tasks.rados.rados.0.smithi026.stdout:2434: done (0 left) 2024-09-13T14:27:14.958 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 231 2024-09-13T14:27:14.958 INFO:tasks.rados.rados.0.smithi026.stdout:2435: seq_num 765 ranges {4366336=417792,4784128=401408} 2024-09-13T14:27:14.984 INFO:tasks.rados.rados.0.smithi026.stdout:2435: writing smithi02683976-39 from 4366336 to 4784128 tid 1 2024-09-13T14:27:14.989 INFO:tasks.rados.rados.0.smithi026.stdout:2435: writing smithi02683976-39 from 4784128 to 5185536 tid 2 2024-09-13T14:27:14.989 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 231 2024-09-13T14:27:14.989 INFO:tasks.rados.rados.0.smithi026.stdout:2436: seq_num 766 ranges {5251072=434176,5685248=401408} 2024-09-13T14:27:15.018 INFO:tasks.rados.rados.0.smithi026.stdout:2436: writing smithi02683976-13 from 5251072 to 5685248 tid 1 2024-09-13T14:27:15.020 INFO:tasks.rados.rados.0.smithi026.stdout:2436: writing smithi02683976-13 from 5685248 to 6086656 tid 2 2024-09-13T14:27:15.020 INFO:tasks.rados.rados.0.smithi026.stdout:2437: snap_remove snap 228 2024-09-13T14:27:15.705 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:27:15.395+0000 7f7686525640 -1 Falling back to public interface 2024-09-13T14:27:15.975 INFO:tasks.rados.rados.0.smithi026.stdout:2437: done (2 left) 2024-09-13T14:27:15.975 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 231 2024-09-13T14:27:15.975 INFO:tasks.rados.rados.0.smithi026.stdout:2438: seq_num 767 ranges {0=450560,450560=401408} 2024-09-13T14:27:15.977 INFO:tasks.rados.rados.0.smithi026.stdout:2438: writing smithi02683976-28 from 0 to 450560 tid 1 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2438: writing smithi02683976-28 from 450560 to 851968 tid 2 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2439: read oid 31 snap 225 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2439: expect (ObjNum 742 snap 221 seq_num 742) 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2440: snap_create 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2435: finishing write tid 1 to smithi02683976-39 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2435: finishing write tid 2 to smithi02683976-39 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2435: finishing write tid 3 to smithi02683976-39 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2435: finishing write tid 4 to smithi02683976-39 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1033 (ObjNum 765 snap 231 seq_num 765) dirty exists 2024-09-13T14:27:15.981 INFO:tasks.rados.rados.0.smithi026.stdout:2435: left oid 39 (ObjNum 765 snap 231 seq_num 765) 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:2436: finishing write tid 1 to smithi02683976-13 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:2436: finishing write tid 2 to smithi02683976-13 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:2436: finishing write tid 3 to smithi02683976-13 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:2436: finishing write tid 4 to smithi02683976-13 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1148 (ObjNum 766 snap 231 seq_num 766) dirty exists 2024-09-13T14:27:15.982 INFO:tasks.rados.rados.0.smithi026.stdout:2436: left oid 13 (ObjNum 766 snap 231 seq_num 766) 2024-09-13T14:27:15.983 INFO:tasks.rados.rados.0.smithi026.stdout:2438: finishing write tid 1 to smithi02683976-28 2024-09-13T14:27:15.984 INFO:tasks.rados.rados.0.smithi026.stdout:2438: finishing write tid 2 to smithi02683976-28 2024-09-13T14:27:15.984 INFO:tasks.rados.rados.0.smithi026.stdout:2438: finishing write tid 3 to smithi02683976-28 2024-09-13T14:27:15.986 INFO:tasks.rados.rados.0.smithi026.stdout:2438: finishing write tid 4 to smithi02683976-28 2024-09-13T14:27:15.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1153 (ObjNum 767 snap 231 seq_num 767) dirty exists 2024-09-13T14:27:15.986 INFO:tasks.rados.rados.0.smithi026.stdout:2438: left oid 28 (ObjNum 767 snap 231 seq_num 767) 2024-09-13T14:27:16.014 INFO:tasks.rados.rados.0.smithi026.stdout:2439: expect (ObjNum 742 snap 221 seq_num 742) 2024-09-13T14:27:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:15 smithi145 ceph-mon[89145]: pgmap v404: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.0 MiB/s wr, 2 op/s; 24/240 objects degraded (10.000%) 2024-09-13T14:27:16.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:15 smithi145 ceph-mon[89145]: osdmap e537: 8 total, 7 up, 8 in 2024-09-13T14:27:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:15 smithi026 ceph-mon[97022]: pgmap v404: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.0 MiB/s wr, 2 op/s; 24/240 objects degraded (10.000%) 2024-09-13T14:27:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:15 smithi026 ceph-mon[97022]: osdmap e537: 8 total, 7 up, 8 in 2024-09-13T14:27:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:15 smithi026 ceph-mon[99461]: pgmap v404: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 169 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.0 MiB/s wr, 2 op/s; 24/240 objects degraded (10.000%) 2024-09-13T14:27:16.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:15 smithi026 ceph-mon[99461]: osdmap e537: 8 total, 7 up, 8 in 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2435: done (4 left) 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2436: done (3 left) 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2438: done (2 left) 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2439: done (1 left) 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2440: done (0 left) 2024-09-13T14:27:16.979 INFO:tasks.rados.rados.0.smithi026.stdout:2441: setattr oid 23 current snap is 232 2024-09-13T14:27:16.980 INFO:tasks.rados.rados.0.smithi026.stdout:2442: snap_create 2024-09-13T14:27:16.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1111 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:27:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:16 smithi026 ceph-mon[97022]: osdmap e538: 8 total, 7 up, 8 in 2024-09-13T14:27:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:16 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 24/255 objects degraded (9.412%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:16 smithi026 ceph-mon[99461]: osdmap e538: 8 total, 7 up, 8 in 2024-09-13T14:27:17.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:16 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 24/255 objects degraded (9.412%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:16 smithi145 ceph-mon[89145]: osdmap e538: 8 total, 7 up, 8 in 2024-09-13T14:27:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:16 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 24/255 objects degraded (9.412%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2441: done (1 left) 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2442: done (0 left) 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2443: read oid 30 snap 230 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2443: expect deleted 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2444: read oid 25 snap -1 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2444: expect deleted 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2445: read oid 50 snap -1 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2445: expect deleted 2024-09-13T14:27:17.982 INFO:tasks.rados.rados.0.smithi026.stdout:2446: snap_create 2024-09-13T14:27:18.088 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:17 smithi145 ceph-mon[89145]: pgmap v407: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 172 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 752 KiB/s wr, 6 op/s; 24/255 objects degraded (9.412%) 2024-09-13T14:27:18.088 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:17 smithi145 ceph-mon[89145]: osdmap e539: 8 total, 7 up, 8 in 2024-09-13T14:27:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:17 smithi026 ceph-mon[97022]: pgmap v407: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 172 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 752 KiB/s wr, 6 op/s; 24/255 objects degraded (9.412%) 2024-09-13T14:27:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:17 smithi026 ceph-mon[97022]: osdmap e539: 8 total, 7 up, 8 in 2024-09-13T14:27:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:17 smithi026 ceph-mon[99461]: pgmap v407: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 172 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 752 KiB/s wr, 6 op/s; 24/255 objects degraded (9.412%) 2024-09-13T14:27:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:17 smithi026 ceph-mon[99461]: osdmap e539: 8 total, 7 up, 8 in 2024-09-13T14:27:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:2443: done (3 left) 2024-09-13T14:27:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:2444: done (2 left) 2024-09-13T14:27:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:2445: done (1 left) 2024-09-13T14:27:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:2446: done (0 left) 2024-09-13T14:27:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:2447: snap_remove snap 226 2024-09-13T14:27:19.281 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:27:19.079+0000 7f7686525640 -1 osd.6 529 log_to_monitors true 2024-09-13T14:27:19.281 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:18 smithi145 ceph-mon[89145]: osdmap e540: 8 total, 7 up, 8 in 2024-09-13T14:27:19.281 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:18 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.281 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:18 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.281 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:18 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.281 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:18 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[97022]: osdmap e540: 8 total, 7 up, 8 in 2024-09-13T14:27:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[99461]: osdmap e540: 8 total, 7 up, 8 in 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:18 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:19.996 INFO:tasks.rados.rados.0.smithi026.stdout:2447: done (0 left) 2024-09-13T14:27:19.996 INFO:tasks.rados.rados.0.smithi026.stdout:2448: rmattr oid 49 current snap is 234 2024-09-13T14:27:19.996 INFO:tasks.rados.rados.0.smithi026.stdout:2449: rollback oid 46 current snap is 234 2024-09-13T14:27:19.996 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 232 2024-09-13T14:27:19.996 INFO:tasks.rados.rados.0.smithi026.stdout:2450: snap_create 2024-09-13T14:27:20.002 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1154 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:27:20.006 INFO:tasks.rados.rados.0.smithi026.stdout:2449: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:27:20.006 INFO:tasks.rados.rados.0.smithi026.stdout:2449: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:27:20.007 INFO:tasks.rados.rados.0.smithi026.stdout:2449: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:27:20.007 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 938 (ObjNum 716 snap 209 seq_num 716) dirty exists 2024-09-13T14:27:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[97022]: pgmap v410: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 173 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 952 KiB/s wr, 7 op/s; 24/258 objects degraded (9.302%) 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[97022]: osdmap e541: 8 total, 7 up, 8 in 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[97022]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[97022]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[99461]: pgmap v410: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 173 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 952 KiB/s wr, 7 op/s; 24/258 objects degraded (9.302%) 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[99461]: osdmap e541: 8 total, 7 up, 8 in 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[99461]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:19 smithi026 ceph-mon[99461]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:27:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:27:20.027+0000 7f767e2cf640 -1 osd.6 529 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:27:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:19 smithi145 ceph-mon[89145]: pgmap v410: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 173 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 952 KiB/s wr, 7 op/s; 24/258 objects degraded (9.302%) 2024-09-13T14:27:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:19 smithi145 ceph-mon[89145]: osdmap e541: 8 total, 7 up, 8 in 2024-09-13T14:27:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:19 smithi145 ceph-mon[89145]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:19 smithi145 ceph-mon[89145]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T14:27:20.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:2448: done (2 left) 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:2449: done (1 left) 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:2450: done (0 left) 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:2451: rmattr oid 32 current snap is 235 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:2452: rollback oid 9 current snap is 235 2024-09-13T14:27:21.010 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 224 2024-09-13T14:27:21.011 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 235 2024-09-13T14:27:21.011 INFO:tasks.rados.rados.0.smithi026.stdout:2453: seq_num 768 ranges {5046272=466944,5513216=401408} 2024-09-13T14:27:21.041 INFO:tasks.rados.rados.0.smithi026.stdout:2453: writing smithi02683976-7 from 5046272 to 5513216 tid 1 2024-09-13T14:27:21.043 INFO:tasks.rados.rados.0.smithi026.stdout:2453: writing smithi02683976-7 from 5513216 to 5914624 tid 2 2024-09-13T14:27:21.044 INFO:tasks.rados.rados.0.smithi026.stdout:2454: snap_create 2024-09-13T14:27:21.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1115 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:27:21.046 INFO:tasks.rados.rados.0.smithi026.stdout:2453: finishing write tid 1 to smithi02683976-7 2024-09-13T14:27:21.046 INFO:tasks.rados.rados.0.smithi026.stdout:2453: finishing write tid 2 to smithi02683976-7 2024-09-13T14:27:21.046 INFO:tasks.rados.rados.0.smithi026.stdout:2453: finishing write tid 3 to smithi02683976-7 2024-09-13T14:27:21.047 INFO:tasks.rados.rados.0.smithi026.stdout:2453: finishing write tid 4 to smithi02683976-7 2024-09-13T14:27:21.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1128 (ObjNum 768 snap 235 seq_num 768) dirty exists 2024-09-13T14:27:21.047 INFO:tasks.rados.rados.0.smithi026.stdout:2453: left oid 7 (ObjNum 768 snap 235 seq_num 768) 2024-09-13T14:27:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:21 smithi145 ceph-mon[89145]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:27:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:21 smithi145 ceph-mon[89145]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:21 smithi145 ceph-mon[89145]: osdmap e542: 8 total, 7 up, 8 in 2024-09-13T14:27:21.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:21 smithi145 ceph-mon[89145]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[97022]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:27:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[97022]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[97022]: osdmap e542: 8 total, 7 up, 8 in 2024-09-13T14:27:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[97022]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[99461]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:27:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[99461]: from='osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[99461]: osdmap e542: 8 total, 7 up, 8 in 2024-09-13T14:27:21.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-mon[99461]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:21] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.51.0" 2024-09-13T14:27:22.014 INFO:tasks.rados.rados.0.smithi026.stdout:2451: done (3 left) 2024-09-13T14:27:22.014 INFO:tasks.rados.rados.0.smithi026.stdout:2453: done (2 left) 2024-09-13T14:27:22.014 INFO:tasks.rados.rados.0.smithi026.stdout:2454: done (1 left) 2024-09-13T14:27:22.014 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 236 2024-09-13T14:27:22.014 INFO:tasks.rados.rados.0.smithi026.stdout:2455: seq_num 769 ranges {7954432=483328,8437760=401408} 2024-09-13T14:27:22.061 INFO:tasks.rados.rados.0.smithi026.stdout:2455: writing smithi02683976-1 from 7954432 to 8437760 tid 1 2024-09-13T14:27:22.063 INFO:tasks.rados.rados.0.smithi026.stdout:2455: writing smithi02683976-1 from 8437760 to 8839168 tid 2 2024-09-13T14:27:22.064 INFO:tasks.rados.rados.0.smithi026.stdout:2456: snap_create 2024-09-13T14:27:22.064 INFO:tasks.rados.rados.0.smithi026.stdout:2452: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:27:22.064 INFO:tasks.rados.rados.0.smithi026.stdout:2452: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:27:22.064 INFO:tasks.rados.rados.0.smithi026.stdout:2452: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:27:22.064 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1199 (ObjNum 638 snap 180 seq_num 638) dirty exists 2024-09-13T14:27:22.065 INFO:tasks.rados.rados.0.smithi026.stdout:2455: finishing write tid 1 to smithi02683976-1 2024-09-13T14:27:22.066 INFO:tasks.rados.rados.0.smithi026.stdout:2455: finishing write tid 2 to smithi02683976-1 2024-09-13T14:27:22.066 INFO:tasks.rados.rados.0.smithi026.stdout:2455: finishing write tid 3 to smithi02683976-1 2024-09-13T14:27:22.067 INFO:tasks.rados.rados.0.smithi026.stdout:2455: finishing write tid 4 to smithi02683976-1 2024-09-13T14:27:22.068 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1275 (ObjNum 769 snap 236 seq_num 769) dirty exists 2024-09-13T14:27:22.068 INFO:tasks.rados.rados.0.smithi026.stdout:2455: left oid 1 (ObjNum 769 snap 236 seq_num 769) 2024-09-13T14:27:22.148 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:22 smithi145 ceph-mon[89145]: pgmap v413: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 408 KiB/s wr, 6 op/s; 24/264 objects degraded (9.091%) 2024-09-13T14:27:22.149 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:22 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:22.149 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:22 smithi145 ceph-mon[89145]: osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697] boot 2024-09-13T14:27:22.149 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:22 smithi145 ceph-mon[89145]: osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:27:22.149 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:22 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[99461]: pgmap v413: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 408 KiB/s wr, 6 op/s; 24/264 objects degraded (9.091%) 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[99461]: osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697] boot 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[99461]: osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[97022]: pgmap v413: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 408 KiB/s wr, 6 op/s; 24/264 objects degraded (9.091%) 2024-09-13T14:27:22.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:22.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[97022]: osd.6 [v2:172.21.15.145:6816/3319036697,v1:172.21.15.145:6817/3319036697] boot 2024-09-13T14:27:22.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[97022]: osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:27:22.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:27:22.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:27:22.305+0000 7f82de253640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-13T14:27:23.017 INFO:tasks.rados.rados.0.smithi026.stdout:2452: done (2 left) 2024-09-13T14:27:23.018 INFO:tasks.rados.rados.0.smithi026.stdout:2455: done (1 left) 2024-09-13T14:27:23.018 INFO:tasks.rados.rados.0.smithi026.stdout:2456: done (0 left) 2024-09-13T14:27:23.018 INFO:tasks.rados.rados.0.smithi026.stdout:2457: delete oid 46 current snap is 237 2024-09-13T14:27:23.024 INFO:tasks.rados.rados.0.smithi026.stdout:2457: done (0 left) 2024-09-13T14:27:23.024 INFO:tasks.rados.rados.0.smithi026.stdout:2458: delete oid 40 current snap is 237 2024-09-13T14:27:23.029 INFO:tasks.rados.rados.0.smithi026.stdout:2458: done (0 left) 2024-09-13T14:27:23.029 INFO:tasks.rados.rados.0.smithi026.stdout:2459: snap_create 2024-09-13T14:27:23.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:23.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[97022]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:23.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:23.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:23 smithi026 ceph-mon[99461]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-13T14:27:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:27:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:23.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:23 smithi145 ceph-mon[89145]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-13T14:27:24.024 INFO:tasks.rados.rados.0.smithi026.stdout:2459: done (0 left) 2024-09-13T14:27:24.024 INFO:tasks.rados.rados.0.smithi026.stdout:2460: read oid 50 snap -1 2024-09-13T14:27:24.024 INFO:tasks.rados.rados.0.smithi026.stdout:2460: expect deleted 2024-09-13T14:27:24.024 INFO:tasks.rados.rados.0.smithi026.stdout:2461: read oid 18 snap -1 2024-09-13T14:27:24.024 INFO:tasks.rados.rados.0.smithi026.stdout:2461: expect deleted 2024-09-13T14:27:24.025 INFO:tasks.rados.rados.0.smithi026.stdout:2462: rmattr oid 13 current snap is 238 2024-09-13T14:27:24.025 INFO:tasks.rados.rados.0.smithi026.stdout:2463: rmattr oid 5 current snap is 238 2024-09-13T14:27:24.025 INFO:tasks.rados.rados.0.smithi026.stdout:2464: snap_create 2024-09-13T14:27:24.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1277 (ObjNum 706 snap 207 seq_num 706) dirty exists 2024-09-13T14:27:24.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1154 (ObjNum 766 snap 231 seq_num 766) dirty exists 2024-09-13T14:27:24.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[97022]: pgmap v416: 121 pgs: 10 peering, 28 active+undersized, 4 active+undersized+degraded, 79 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 208 KiB/s wr, 5 op/s; 12/264 objects degraded (4.545%) 2024-09-13T14:27:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[97022]: osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:27:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 12/264 objects degraded (4.545%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[99461]: pgmap v416: 121 pgs: 10 peering, 28 active+undersized, 4 active+undersized+degraded, 79 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 208 KiB/s wr, 5 op/s; 12/264 objects degraded (4.545%) 2024-09-13T14:27:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[99461]: osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:27:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:24 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 12/264 objects degraded (4.545%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:24 smithi145 ceph-mon[89145]: pgmap v416: 121 pgs: 10 peering, 28 active+undersized, 4 active+undersized+degraded, 79 active+clean; 174 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 208 KiB/s wr, 5 op/s; 12/264 objects degraded (4.545%) 2024-09-13T14:27:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:24 smithi145 ceph-mon[89145]: osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:27:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:24 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 12/264 objects degraded (4.545%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2460: done (4 left) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2461: done (3 left) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2462: done (2 left) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2463: done (1 left) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2464: done (0 left) 2024-09-13T14:27:25.030 INFO:tasks.rados.rados.0.smithi026.stdout:2465: copy_from oid 48 from oid 32 current snap is 239 2024-09-13T14:27:25.031 INFO:tasks.rados.rados.0.smithi026.stdout:2466: rollback oid 19 current snap is 239 2024-09-13T14:27:25.031 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 235 2024-09-13T14:27:25.031 INFO:tasks.rados.rados.0.smithi026.stdout:2467: copy_from oid 36 from oid 37 current snap is 239 2024-09-13T14:27:25.031 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 239 2024-09-13T14:27:25.031 INFO:tasks.rados.rados.0.smithi026.stdout:2468: seq_num 772 ranges {5185536=532480,5718016=401408} 2024-09-13T14:27:25.063 INFO:tasks.rados.rados.0.smithi026.stdout:2468: writing smithi02683976-39 from 5185536 to 5718016 tid 1 2024-09-13T14:27:25.065 INFO:tasks.rados.rados.0.smithi026.stdout:2468: writing smithi02683976-39 from 5718016 to 6119424 tid 2 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:2469: snap_create 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:2467: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:2466: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:2467: finishing copy_from to smithi02683976-36 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1021 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:27:25.066 INFO:tasks.rados.rados.0.smithi026.stdout:2465: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:27:25.070 INFO:tasks.rados.rados.0.smithi026.stdout:2468: finishing write tid 1 to smithi02683976-39 2024-09-13T14:27:25.070 INFO:tasks.rados.rados.0.smithi026.stdout:2465: finishing copy_from to smithi02683976-48 2024-09-13T14:27:25.070 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1115 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:27:25.070 INFO:tasks.rados.rados.0.smithi026.stdout:2468: finishing write tid 2 to smithi02683976-39 2024-09-13T14:27:25.070 INFO:tasks.rados.rados.0.smithi026.stdout:2468: finishing write tid 3 to smithi02683976-39 2024-09-13T14:27:25.071 INFO:tasks.rados.rados.0.smithi026.stdout:2468: finishing write tid 4 to smithi02683976-39 2024-09-13T14:27:25.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1036 (ObjNum 772 snap 239 seq_num 772) dirty exists 2024-09-13T14:27:25.071 INFO:tasks.rados.rados.0.smithi026.stdout:2468: left oid 39 (ObjNum 772 snap 239 seq_num 772) 2024-09-13T14:27:25.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:25 smithi026 ceph-mon[99461]: osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:27:25.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:25 smithi026 ceph-mon[97022]: osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:27:25.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:25 smithi145 ceph-mon[89145]: osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2465: done (4 left) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2466: done (3 left) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2467: done (2 left) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2468: done (1 left) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2469: done (0 left) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2470: read oid 28 snap -1 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2470: expect (ObjNum 767 snap 231 seq_num 767) 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2471: read oid 50 snap -1 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2471: expect deleted 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 240 2024-09-13T14:27:26.037 INFO:tasks.rados.rados.0.smithi026.stdout:2472: seq_num 773 ranges {598016=548864,1146880=401408} 2024-09-13T14:27:26.042 INFO:tasks.rados.rados.0.smithi026.stdout:2472: writing smithi02683976-15 from 598016 to 1146880 tid 1 2024-09-13T14:27:26.044 INFO:tasks.rados.rados.0.smithi026.stdout:2472: writing smithi02683976-15 from 1146880 to 1548288 tid 2 2024-09-13T14:27:26.044 INFO:tasks.rados.rados.0.smithi026.stdout:2473: snap_remove snap 234 2024-09-13T14:27:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:26 smithi026 ceph-mon[97022]: pgmap v419: 121 pgs: 10 peering, 22 active+undersized, 4 active+undersized+degraded, 85 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 212 KiB/s wr, 0 op/s; 12/267 objects degraded (4.494%) 2024-09-13T14:27:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:26 smithi026 ceph-mon[97022]: osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:27:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:26 smithi026 ceph-mon[99461]: pgmap v419: 121 pgs: 10 peering, 22 active+undersized, 4 active+undersized+degraded, 85 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 212 KiB/s wr, 0 op/s; 12/267 objects degraded (4.494%) 2024-09-13T14:27:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:26 smithi026 ceph-mon[99461]: osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:27:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:26 smithi145 ceph-mon[89145]: pgmap v419: 121 pgs: 10 peering, 22 active+undersized, 4 active+undersized+degraded, 85 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 212 KiB/s wr, 0 op/s; 12/267 objects degraded (4.494%) 2024-09-13T14:27:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:26 smithi145 ceph-mon[89145]: osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:27:27.040 INFO:tasks.rados.rados.0.smithi026.stdout:2470: expect (ObjNum 767 snap 231 seq_num 767) 2024-09-13T14:27:27.065 INFO:tasks.rados.rados.0.smithi026.stdout:2472: finishing write tid 1 to smithi02683976-15 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2472: finishing write tid 2 to smithi02683976-15 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2472: finishing write tid 3 to smithi02683976-15 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2472: finishing write tid 4 to smithi02683976-15 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1034 (ObjNum 773 snap 240 seq_num 773) dirty exists 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2472: left oid 15 (ObjNum 773 snap 240 seq_num 773) 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2470: done (3 left) 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2471: done (2 left) 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2472: done (1 left) 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2473: done (0 left) 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 240 2024-09-13T14:27:27.066 INFO:tasks.rados.rados.0.smithi026.stdout:2474: seq_num 774 ranges {0=565248,565248=401408} 2024-09-13T14:27:27.068 INFO:tasks.rados.rados.0.smithi026.stdout:2474: writing smithi02683976-41 from 0 to 565248 tid 1 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2474: writing smithi02683976-41 from 565248 to 966656 tid 2 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2475: rollback oid 31 current snap is 240 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 233 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2476: read oid 24 snap 232 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2476: expect deleted 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2477: read oid 13 snap 225 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2477: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:27:27.071 INFO:tasks.rados.rados.0.smithi026.stdout:2478: delete oid 1 current snap is 240 2024-09-13T14:27:27.079 INFO:tasks.rados.rados.0.smithi026.stdout:2475: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:27:27.079 INFO:tasks.rados.rados.0.smithi026.stdout:2475: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:27:27.079 INFO:tasks.rados.rados.0.smithi026.stdout:2474: finishing write tid 1 to smithi02683976-41 2024-09-13T14:27:27.079 INFO:tasks.rados.rados.0.smithi026.stdout:2474: finishing write tid 2 to smithi02683976-41 2024-09-13T14:27:27.079 INFO:tasks.rados.rados.0.smithi026.stdout:2474: finishing write tid 3 to smithi02683976-41 2024-09-13T14:27:27.080 INFO:tasks.rados.rados.0.smithi026.stdout:2475: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:27:27.080 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1211 (ObjNum 742 snap 221 seq_num 742) dirty exists 2024-09-13T14:27:27.080 INFO:tasks.rados.rados.0.smithi026.stdout:2474: finishing write tid 4 to smithi02683976-41 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 837 (ObjNum 774 snap 240 seq_num 774) dirty exists 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2474: left oid 41 (ObjNum 774 snap 240 seq_num 774) 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2474: done (4 left) 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2475: done (3 left) 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2476: done (2 left) 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2478: done (1 left) 2024-09-13T14:27:27.081 INFO:tasks.rados.rados.0.smithi026.stdout:2479: snap_create 2024-09-13T14:27:27.093 INFO:tasks.rados.rados.0.smithi026.stdout:2477: expect (ObjNum 573 snap 163 seq_num 573) 2024-09-13T14:27:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[97022]: osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:27:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[99461]: osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:27:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:27 smithi145 ceph-mon[89145]: osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:27:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:28.043 INFO:tasks.rados.rados.0.smithi026.stdout:2477: done (1 left) 2024-09-13T14:27:28.043 INFO:tasks.rados.rados.0.smithi026.stdout:2479: done (0 left) 2024-09-13T14:27:28.044 INFO:tasks.rados.rados.0.smithi026.stdout:2480: snap_create 2024-09-13T14:27:28.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[99461]: pgmap v422: 121 pgs: 121 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 428 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:28.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[99461]: osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/267 objects degraded (4.494%), 4 pgs degraded) 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[97022]: pgmap v422: 121 pgs: 121 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 428 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[97022]: osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/267 objects degraded (4.494%), 4 pgs degraded) 2024-09-13T14:27:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:28 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:27:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:28 smithi145 ceph-mon[89145]: pgmap v422: 121 pgs: 121 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 428 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:28 smithi145 ceph-mon[89145]: osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:27:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:28 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/267 objects degraded (4.494%), 4 pgs degraded) 2024-09-13T14:27:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:28 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:27:28.957 INFO:tasks.rados.rados.0.smithi026.stdout:2480: done (0 left) 2024-09-13T14:27:28.957 INFO:tasks.rados.rados.0.smithi026.stdout:2481: delete oid 49 current snap is 242 2024-09-13T14:27:28.961 INFO:tasks.rados.rados.0.smithi026.stdout:2481: done (0 left) 2024-09-13T14:27:28.961 INFO:tasks.rados.rados.0.smithi026.stdout:2482: delete oid 41 current snap is 242 2024-09-13T14:27:28.964 INFO:tasks.rados.rados.0.smithi026.stdout:2482: done (0 left) 2024-09-13T14:27:28.964 INFO:tasks.rados.rados.0.smithi026.stdout:2483: copy_from oid 45 from oid 18 current snap is 242 2024-09-13T14:27:28.964 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 242 2024-09-13T14:27:28.964 INFO:tasks.rados.rados.0.smithi026.stdout:2484: seq_num 778 ranges {614400=638976,1253376=401408} 2024-09-13T14:27:28.971 INFO:tasks.rados.rados.0.smithi026.stdout:2484: writing smithi02683976-6 from 614400 to 1253376 tid 1 2024-09-13T14:27:28.973 INFO:tasks.rados.rados.0.smithi026.stdout:2484: writing smithi02683976-6 from 1253376 to 1654784 tid 2 2024-09-13T14:27:28.973 INFO:tasks.rados.rados.0.smithi026.stdout:2485: copy_from oid 41 from oid 11 current snap is 242 2024-09-13T14:27:28.973 INFO:tasks.rados.rados.0.smithi026.stdout:2483: finishing copy_from to smithi02683976-45 2024-09-13T14:27:28.974 INFO:tasks.rados.rados.0.smithi026.stdout:2483: got expected ENOENT (src dne) 2024-09-13T14:27:28.974 INFO:tasks.rados.rados.0.smithi026.stdout:2483: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:27:28.974 INFO:tasks.rados.rados.0.smithi026.stdout:2483: done (2 left) 2024-09-13T14:27:28.974 INFO:tasks.rados.rados.0.smithi026.stdout:2486: setattr oid 30 current snap is 242 2024-09-13T14:27:28.974 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 242 2024-09-13T14:27:28.975 INFO:tasks.rados.rados.0.smithi026.stdout:2487: seq_num 779 ranges {0=655360,655360=401408} 2024-09-13T14:27:28.978 INFO:tasks.rados.rados.0.smithi026.stdout:2487: writing smithi02683976-47 from 0 to 655360 tid 1 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2487: writing smithi02683976-47 from 655360 to 1056768 tid 2 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2488: rollback oid 32 current snap is 242 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 235 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2484: finishing write tid 1 to smithi02683976-6 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2484: finishing write tid 2 to smithi02683976-6 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2484: finishing write tid 3 to smithi02683976-6 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1035 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:27:28.982 INFO:tasks.rados.rados.0.smithi026.stdout:2486: done (4 left) 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2489: setattr oid 17 current snap is 242 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2485: finishing copy_from to smithi02683976-41 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2485: got expected ENOENT (src dne) 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2484: finishing write tid 4 to smithi02683976-6 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1215 (ObjNum 778 snap 242 seq_num 778) dirty exists 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2484: left oid 6 (ObjNum 778 snap 242 seq_num 778) 2024-09-13T14:27:28.983 INFO:tasks.rados.rados.0.smithi026.stdout:2485: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:27:28.984 INFO:tasks.rados.rados.0.smithi026.stdout:2484: done (4 left) 2024-09-13T14:27:28.984 INFO:tasks.rados.rados.0.smithi026.stdout:2485: done (3 left) 2024-09-13T14:27:28.984 INFO:tasks.rados.rados.0.smithi026.stdout:2490: copy_from oid 19 from oid 4 current snap is 242 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2491: rmattr oid 6 current snap is 242 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2491: done (4 left) 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2492: copy_from oid 30 from oid 48 current snap is 242 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2493: read oid 43 snap -1 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2493: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2494: rmattr oid 49 current snap is 242 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2494: done (6 left) 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2495: delete oid 38 current snap is 242 2024-09-13T14:27:28.985 INFO:tasks.rados.rados.0.smithi026.stdout:2487: finishing write tid 1 to smithi02683976-47 2024-09-13T14:27:28.986 INFO:tasks.rados.rados.0.smithi026.stdout:2487: finishing write tid 2 to smithi02683976-47 2024-09-13T14:27:28.986 INFO:tasks.rados.rados.0.smithi026.stdout:2487: finishing write tid 3 to smithi02683976-47 2024-09-13T14:27:28.987 INFO:tasks.rados.rados.0.smithi026.stdout:2488: finishing rollback tid 0 to smithi02683976-32 2024-09-13T14:27:28.988 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 839 (ObjNum 762 snap 230 seq_num 762) dirty exists 2024-09-13T14:27:28.989 INFO:tasks.rados.rados.0.smithi026.stdout:2488: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2488: finishing rollback tid 2 to smithi02683976-32 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1121 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2487: finishing write tid 4 to smithi02683976-47 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1118 (ObjNum 779 snap 242 seq_num 779) dirty exists 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2487: left oid 47 (ObjNum 779 snap 242 seq_num 779) 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2487: done (6 left) 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2488: done (5 left) 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2489: done (4 left) 2024-09-13T14:27:28.992 INFO:tasks.rados.rados.0.smithi026.stdout:2495: done (3 left) 2024-09-13T14:27:28.993 INFO:tasks.rados.rados.0.smithi026.stdout:2496: snap_remove snap 237 2024-09-13T14:27:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:29 smithi026 ceph-mon[99461]: osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:27:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:29 smithi026 ceph-mon[99461]: osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:27:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:29 smithi026 ceph-mon[97022]: osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:27:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:29 smithi026 ceph-mon[97022]: osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:27:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:29 smithi145 ceph-mon[89145]: osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:27:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:29 smithi145 ceph-mon[89145]: osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:27:29.960 INFO:tasks.rados.rados.0.smithi026.stdout:2493: expect (ObjNum 660 snap 188 seq_num 660) 2024-09-13T14:27:30.034 INFO:tasks.rados.rados.0.smithi026.stdout:2490: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2490: finishing copy_from to smithi02683976-19 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1029 (ObjNum 747 snap 224 seq_num 747) dirty exists 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2492: finishing copy_from racing read to smithi02683976-30 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2492: finishing copy_from to smithi02683976-30 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1115 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2490: done (3 left) 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2492: done (2 left) 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2493: done (1 left) 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2496: done (0 left) 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 242 2024-09-13T14:27:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:2497: seq_num 781 ranges {4366336=688128,5054464=401408} 2024-09-13T14:27:30.062 INFO:tasks.rados.rados.0.smithi026.stdout:2497: writing smithi02683976-4 from 4366336 to 5054464 tid 1 2024-09-13T14:27:30.064 INFO:tasks.rados.rados.0.smithi026.stdout:2497: writing smithi02683976-4 from 5054464 to 5455872 tid 2 2024-09-13T14:27:30.064 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 242 2024-09-13T14:27:30.064 INFO:tasks.rados.rados.0.smithi026.stdout:2498: seq_num 782 ranges {6086656=704512,6791168=401408} 2024-09-13T14:27:30.101 INFO:tasks.rados.rados.0.smithi026.stdout:2498: writing smithi02683976-13 from 6086656 to 6791168 tid 1 2024-09-13T14:27:30.104 INFO:tasks.rados.rados.0.smithi026.stdout:2498: writing smithi02683976-13 from 6791168 to 7192576 tid 2 2024-09-13T14:27:30.104 INFO:tasks.rados.rados.0.smithi026.stdout:2499: snap_remove snap 232 2024-09-13T14:27:30.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:30 smithi026 ceph-mon[99461]: pgmap v425: 121 pgs: 121 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 444 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:30 smithi026 ceph-mon[99461]: osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:27:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:30 smithi026 ceph-mon[97022]: pgmap v425: 121 pgs: 121 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 444 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:30 smithi026 ceph-mon[97022]: osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:27:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:30 smithi145 ceph-mon[89145]: pgmap v425: 121 pgs: 121 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 444 KiB/s wr, 13 op/s; 5.6 MiB/s, 1 objects/s recovering 2024-09-13T14:27:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:30 smithi145 ceph-mon[89145]: osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:27:30.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2499: done (2 left) 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2500: setattr oid 10 current snap is 242 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2497: finishing write tid 1 to smithi02683976-4 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2497: finishing write tid 2 to smithi02683976-4 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2497: finishing write tid 3 to smithi02683976-4 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:2497: finishing write tid 4 to smithi02683976-4 2024-09-13T14:27:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1118 (ObjNum 781 snap 242 seq_num 781) dirty exists 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2497: left oid 4 (ObjNum 781 snap 242 seq_num 781) 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2498: finishing write tid 1 to smithi02683976-13 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2498: finishing write tid 2 to smithi02683976-13 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2498: finishing write tid 3 to smithi02683976-13 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2498: finishing write tid 4 to smithi02683976-13 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1164 (ObjNum 782 snap 242 seq_num 782) dirty exists 2024-09-13T14:27:31.059 INFO:tasks.rados.rados.0.smithi026.stdout:2498: left oid 13 (ObjNum 782 snap 242 seq_num 782) 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2497: done (2 left) 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2498: done (1 left) 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2501: rmattr oid 39 current snap is 242 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2501: done (1 left) 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2502: rollback oid 23 current snap is 242 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 23 to 231 2024-09-13T14:27:31.060 INFO:tasks.rados.rados.0.smithi026.stdout:2503: snap_remove snap 236 2024-09-13T14:27:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:31] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.51.0" 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:2503: done (2 left) 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:2504: delete oid 19 current snap is 242 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1157 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:2502: finishing rollback tid 0 to smithi02683976-23 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:2502: finishing rollback tid 1 to smithi02683976-23 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:2502: finishing rollback tid 2 to smithi02683976-23 2024-09-13T14:27:32.064 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1131 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:27:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2500: done (2 left) 2024-09-13T14:27:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2502: done (1 left) 2024-09-13T14:27:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2504: done (0 left) 2024-09-13T14:27:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2505: read oid 50 snap -1 2024-09-13T14:27:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:2505: expect deleted 2024-09-13T14:27:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:2506: rollback oid 21 current snap is 242 2024-09-13T14:27:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 240 2024-09-13T14:27:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:2507: delete oid 35 current snap is 242 2024-09-13T14:27:32.076 INFO:tasks.rados.rados.0.smithi026.stdout:2506: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:27:32.076 INFO:tasks.rados.rados.0.smithi026.stdout:2506: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:27:32.077 INFO:tasks.rados.rados.0.smithi026.stdout:2506: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:27:32.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1233 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:32.081 INFO:tasks.rados.rados.0.smithi026.stdout:2505: done (2 left) 2024-09-13T14:27:32.081 INFO:tasks.rados.rados.0.smithi026.stdout:2506: done (1 left) 2024-09-13T14:27:32.081 INFO:tasks.rados.rados.0.smithi026.stdout:2507: done (0 left) 2024-09-13T14:27:32.081 INFO:tasks.rados.rados.0.smithi026.stdout:2508: delete oid 50 current snap is 242 2024-09-13T14:27:32.083 INFO:tasks.rados.rados.0.smithi026.stdout:2508: done (0 left) 2024-09-13T14:27:32.083 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 242 2024-09-13T14:27:32.083 INFO:tasks.rados.rados.0.smithi026.stdout:2509: seq_num 786 ranges {0=770048,770048=401408} 2024-09-13T14:27:32.088 INFO:tasks.rados.rados.0.smithi026.stdout:2509: writing smithi02683976-35 from 0 to 770048 tid 1 2024-09-13T14:27:32.090 INFO:tasks.rados.rados.0.smithi026.stdout:2509: writing smithi02683976-35 from 770048 to 1171456 tid 2 2024-09-13T14:27:32.090 INFO:tasks.rados.rados.0.smithi026.stdout:2510: delete oid 17 current snap is 242 2024-09-13T14:27:32.092 INFO:tasks.rados.rados.0.smithi026.stdout:2510: done (1 left) 2024-09-13T14:27:32.093 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 242 2024-09-13T14:27:32.093 INFO:tasks.rados.rados.0.smithi026.stdout:2511: seq_num 788 ranges {0=802816,802816=401408} 2024-09-13T14:27:32.097 INFO:tasks.rados.rados.0.smithi026.stdout:2511: writing smithi02683976-3 from 0 to 802816 tid 1 2024-09-13T14:27:32.099 INFO:tasks.rados.rados.0.smithi026.stdout:2511: writing smithi02683976-3 from 802816 to 1204224 tid 2 2024-09-13T14:27:32.099 INFO:tasks.rados.rados.0.smithi026.stdout:2512: snap_create 2024-09-13T14:27:32.100 INFO:tasks.rados.rados.0.smithi026.stdout:2509: finishing write tid 1 to smithi02683976-35 2024-09-13T14:27:32.100 INFO:tasks.rados.rados.0.smithi026.stdout:2509: finishing write tid 2 to smithi02683976-35 2024-09-13T14:27:32.100 INFO:tasks.rados.rados.0.smithi026.stdout:2509: finishing write tid 3 to smithi02683976-35 2024-09-13T14:27:32.100 INFO:tasks.rados.rados.0.smithi026.stdout:2511: finishing write tid 1 to smithi02683976-3 2024-09-13T14:27:32.103 INFO:tasks.rados.rados.0.smithi026.stdout:2509: finishing write tid 4 to smithi02683976-35 2024-09-13T14:27:32.103 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1310 (ObjNum 786 snap 242 seq_num 786) dirty exists 2024-09-13T14:27:32.103 INFO:tasks.rados.rados.0.smithi026.stdout:2509: left oid 35 (ObjNum 786 snap 242 seq_num 786) 2024-09-13T14:27:32.103 INFO:tasks.rados.rados.0.smithi026.stdout:2511: finishing write tid 2 to smithi02683976-3 2024-09-13T14:27:32.103 INFO:tasks.rados.rados.0.smithi026.stdout:2511: finishing write tid 3 to smithi02683976-3 2024-09-13T14:27:32.104 INFO:tasks.rados.rados.0.smithi026.stdout:2511: finishing write tid 4 to smithi02683976-3 2024-09-13T14:27:32.104 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1313 (ObjNum 788 snap 242 seq_num 788) dirty exists 2024-09-13T14:27:32.104 INFO:tasks.rados.rados.0.smithi026.stdout:2511: left oid 3 (ObjNum 788 snap 242 seq_num 788) 2024-09-13T14:27:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:32 smithi026 ceph-mon[97022]: pgmap v428: 121 pgs: 121 active+clean; 194 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 625 KiB/s rd, 4.6 MiB/s wr, 18 op/s 2024-09-13T14:27:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:32 smithi026 ceph-mon[97022]: osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:27:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:32 smithi026 ceph-mon[99461]: pgmap v428: 121 pgs: 121 active+clean; 194 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 625 KiB/s rd, 4.6 MiB/s wr, 18 op/s 2024-09-13T14:27:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:32 smithi026 ceph-mon[99461]: osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:27:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:32 smithi145 ceph-mon[89145]: pgmap v428: 121 pgs: 121 active+clean; 194 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 625 KiB/s rd, 4.6 MiB/s wr, 18 op/s 2024-09-13T14:27:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:32 smithi145 ceph-mon[89145]: osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:27:33.069 INFO:tasks.rados.rados.0.smithi026.stdout:2509: done (2 left) 2024-09-13T14:27:33.070 INFO:tasks.rados.rados.0.smithi026.stdout:2511: done (1 left) 2024-09-13T14:27:33.070 INFO:tasks.rados.rados.0.smithi026.stdout:2512: done (0 left) 2024-09-13T14:27:33.070 INFO:tasks.rados.rados.0.smithi026.stdout:2513: delete oid 47 current snap is 243 2024-09-13T14:27:33.074 INFO:tasks.rados.rados.0.smithi026.stdout:2513: done (0 left) 2024-09-13T14:27:33.074 INFO:tasks.rados.rados.0.smithi026.stdout:2514: copy_from oid 37 from oid 34 current snap is 243 2024-09-13T14:27:33.074 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 243 2024-09-13T14:27:33.074 INFO:tasks.rados.rados.0.smithi026.stdout:2515: seq_num 790 ranges {3850240=434176,4284416=548864,4833280=253952} 2024-09-13T14:27:33.098 INFO:tasks.rados.rados.0.smithi026.stdout:2515: writing smithi02683976-9 from 3850240 to 4284416 tid 1 2024-09-13T14:27:33.101 INFO:tasks.rados.rados.0.smithi026.stdout:2515: writing smithi02683976-9 from 4284416 to 4833280 tid 2 2024-09-13T14:27:33.103 INFO:tasks.rados.rados.0.smithi026.stdout:2515: writing smithi02683976-9 from 4833280 to 5087232 tid 3 2024-09-13T14:27:33.103 INFO:tasks.rados.rados.0.smithi026.stdout:2516: rollback oid 27 current snap is 243 2024-09-13T14:27:33.103 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 231 2024-09-13T14:27:33.103 INFO:tasks.rados.rados.0.smithi026.stdout:2517: delete oid 28 current snap is 243 2024-09-13T14:27:33.104 INFO:tasks.rados.rados.0.smithi026.stdout:2515: finishing write tid 1 to smithi02683976-9 2024-09-13T14:27:33.106 INFO:tasks.rados.rados.0.smithi026.stdout:2517: done (3 left) 2024-09-13T14:27:33.106 INFO:tasks.rados.rados.0.smithi026.stdout:2518: delete oid 35 current snap is 243 2024-09-13T14:27:33.107 INFO:tasks.rados.rados.0.smithi026.stdout:2515: finishing write tid 2 to smithi02683976-9 2024-09-13T14:27:33.108 INFO:tasks.rados.rados.0.smithi026.stdout:2515: finishing write tid 3 to smithi02683976-9 2024-09-13T14:27:33.108 INFO:tasks.rados.rados.0.smithi026.stdout:2515: finishing write tid 4 to smithi02683976-9 2024-09-13T14:27:33.108 INFO:tasks.rados.rados.0.smithi026.stdout:2516: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:27:33.112 INFO:tasks.rados.rados.0.smithi026.stdout:2515: finishing write tid 5 to smithi02683976-9 2024-09-13T14:27:33.112 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1239 (ObjNum 790 snap 243 seq_num 790) dirty exists 2024-09-13T14:27:33.112 INFO:tasks.rados.rados.0.smithi026.stdout:2515: left oid 9 (ObjNum 790 snap 243 seq_num 790) 2024-09-13T14:27:33.112 INFO:tasks.rados.rados.0.smithi026.stdout:2516: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:27:33.113 INFO:tasks.rados.rados.0.smithi026.stdout:2516: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:27:33.113 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1317 (ObjNum 755 snap 226 seq_num 755) dirty exists 2024-09-13T14:27:33.115 INFO:tasks.rados.rados.0.smithi026.stdout:2515: done (3 left) 2024-09-13T14:27:33.115 INFO:tasks.rados.rados.0.smithi026.stdout:2516: done (2 left) 2024-09-13T14:27:33.115 INFO:tasks.rados.rados.0.smithi026.stdout:2518: done (1 left) 2024-09-13T14:27:33.115 INFO:tasks.rados.rados.0.smithi026.stdout:2519: snap_create 2024-09-13T14:27:33.119 INFO:tasks.rados.rados.0.smithi026.stdout:2514: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:27:33.120 INFO:tasks.rados.rados.0.smithi026.stdout:2514: finishing copy_from to smithi02683976-37 2024-09-13T14:27:33.121 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 818 (ObjNum 672 snap 189 seq_num 672) dirty exists 2024-09-13T14:27:33.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:33 smithi026 ceph-mon[99461]: osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:27:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:33 smithi026 ceph-mon[97022]: osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:27:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:33 smithi145 ceph-mon[89145]: osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:27:33.959 INFO:tasks.rados.rados.0.smithi026.stdout:2514: done (1 left) 2024-09-13T14:27:33.959 INFO:tasks.rados.rados.0.smithi026.stdout:2519: done (0 left) 2024-09-13T14:27:33.959 INFO:tasks.rados.rados.0.smithi026.stdout:2520: snap_remove snap 224 2024-09-13T14:27:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[99461]: pgmap v431: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.9 MiB/s wr, 22 op/s 2024-09-13T14:27:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[99461]: osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:27:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[99461]: osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:27:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[97022]: pgmap v431: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.9 MiB/s wr, 22 op/s 2024-09-13T14:27:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[97022]: osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:27:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:34 smithi026 ceph-mon[97022]: osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:27:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:34 smithi145 ceph-mon[89145]: pgmap v431: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.9 MiB/s wr, 22 op/s 2024-09-13T14:27:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:34 smithi145 ceph-mon[89145]: osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:27:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:34 smithi145 ceph-mon[89145]: osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:27:34.963 INFO:tasks.rados.rados.0.smithi026.stdout:2520: done (0 left) 2024-09-13T14:27:34.963 INFO:tasks.rados.rados.0.smithi026.stdout:2521: snap_create 2024-09-13T14:27:35.965 INFO:tasks.rados.rados.0.smithi026.stdout:2521: done (0 left) 2024-09-13T14:27:35.965 INFO:tasks.rados.rados.0.smithi026.stdout:2522: snap_remove snap 229 2024-09-13T14:27:36.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:35 smithi145 ceph-mon[89145]: pgmap v434: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T14:27:36.258 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:35 smithi145 ceph-mon[89145]: osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:27:36.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:35 smithi026 ceph-mon[99461]: pgmap v434: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T14:27:36.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:35 smithi026 ceph-mon[99461]: osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:27:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:35 smithi026 ceph-mon[97022]: pgmap v434: 121 pgs: 121 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-13T14:27:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:35 smithi026 ceph-mon[97022]: osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:27:36.968 INFO:tasks.rados.rados.0.smithi026.stdout:2522: done (0 left) 2024-09-13T14:27:36.968 INFO:tasks.rados.rados.0.smithi026.stdout:2523: setattr oid 43 current snap is 245 2024-09-13T14:27:36.971 INFO:tasks.rados.rados.0.smithi026.stdout:2524: setattr oid 29 current snap is 245 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2525: rmattr oid 12 current snap is 245 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2525: done (2 left) 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2526: read oid 3 snap -1 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2526: expect (ObjNum 788 snap 242 seq_num 788) 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2527: copy_from oid 15 from oid 24 current snap is 245 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2528: read oid 32 snap 239 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2528: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2529: read oid 31 snap -1 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:2529: expect (ObjNum 742 snap 221 seq_num 742) 2024-09-13T14:27:36.977 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 245 2024-09-13T14:27:36.978 INFO:tasks.rados.rados.0.smithi026.stdout:2530: seq_num 793 ranges {10051584=483328,10534912=647168,11182080=155648} 2024-09-13T14:27:37.036 INFO:tasks.rados.rados.0.smithi026.stdout:2530: writing smithi02683976-34 from 10051584 to 10534912 tid 1 2024-09-13T14:27:37.039 INFO:tasks.rados.rados.0.smithi026.stdout:2530: writing smithi02683976-34 from 10534912 to 11182080 tid 2 2024-09-13T14:27:37.041 INFO:tasks.rados.rados.0.smithi026.stdout:2530: writing smithi02683976-34 from 11182080 to 11337728 tid 3 2024-09-13T14:27:37.041 INFO:tasks.rados.rados.0.smithi026.stdout:2531: rollback oid 46 current snap is 245 2024-09-13T14:27:37.042 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 225 2024-09-13T14:27:37.042 INFO:tasks.rados.rados.0.smithi026.stdout:2527: finishing copy_from to smithi02683976-15 2024-09-13T14:27:37.042 INFO:tasks.rados.rados.0.smithi026.stdout:2527: got expected ENOENT (src dne) 2024-09-13T14:27:37.042 INFO:tasks.rados.rados.0.smithi026.stdout:2527: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:27:37.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1177 (ObjNum 660 snap 188 seq_num 660) dirty exists 2024-09-13T14:27:37.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1179 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:27:37.043 INFO:tasks.rados.rados.0.smithi026.stdout:2526: expect (ObjNum 788 snap 242 seq_num 788) 2024-09-13T14:27:37.073 INFO:tasks.rados.rados.0.smithi026.stdout:2528: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:36 smithi026 ceph-mon[97022]: osdmap e558: 8 total, 8 up, 8 in 2024-09-13T14:27:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:36 smithi026 ceph-mon[99461]: osdmap e558: 8 total, 8 up, 8 in 2024-09-13T14:27:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:37.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:36 smithi145 ceph-mon[89145]: osdmap e558: 8 total, 8 up, 8 in 2024-09-13T14:27:37.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:37.511 INFO:tasks.rados.rados.0.smithi026.stdout:2529: expect (ObjNum 742 snap 221 seq_num 742) 2024-09-13T14:27:37.939 INFO:tasks.rados.rados.0.smithi026.stdout:2530: finishing write tid 1 to smithi02683976-34 2024-09-13T14:27:37.939 INFO:tasks.rados.rados.0.smithi026.stdout:2531: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2531: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 941 (ObjNum 716 snap 209 seq_num 716) dirty exists 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2530: finishing write tid 2 to smithi02683976-34 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2530: finishing write tid 3 to smithi02683976-34 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2530: finishing write tid 4 to smithi02683976-34 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2523: done (7 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2524: done (6 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2526: done (5 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2527: done (4 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2528: done (3 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2529: done (2 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2531: done (1 left) 2024-09-13T14:27:37.940 INFO:tasks.rados.rados.0.smithi026.stdout:2532: rmattr oid 44 current snap is 245 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:2532: done (1 left) 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 245 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:2530: finishing write tid 5 to smithi02683976-34 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 844 (ObjNum 793 snap 245 seq_num 793) dirty exists 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:2530: left oid 34 (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:27:37.941 INFO:tasks.rados.rados.0.smithi026.stdout:2533: seq_num 794 ranges {1654784=499712,2154496=532480,2686976=270336} 2024-09-13T14:27:37.950 INFO:tasks.rados.rados.0.smithi026.stdout:2533: writing smithi02683976-6 from 1654784 to 2154496 tid 1 2024-09-13T14:27:37.953 INFO:tasks.rados.rados.0.smithi026.stdout:2533: writing smithi02683976-6 from 2154496 to 2686976 tid 2 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2533: writing smithi02683976-6 from 2686976 to 2957312 tid 3 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2530: done (1 left) 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2534: read oid 17 snap -1 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2534: expect deleted 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2535: read oid 7 snap -1 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2535: expect (ObjNum 768 snap 235 seq_num 768) 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 245 2024-09-13T14:27:37.956 INFO:tasks.rados.rados.0.smithi026.stdout:2533: finishing write tid 1 to smithi02683976-6 2024-09-13T14:27:37.957 INFO:tasks.rados.rados.0.smithi026.stdout:2536: seq_num 795 ranges {7438336=524288,7962624=417792,8380416=385024} 2024-09-13T14:27:37.999 INFO:tasks.rados.rados.0.smithi026.stdout:2536: writing smithi02683976-48 from 7438336 to 7962624 tid 1 2024-09-13T14:27:38.001 INFO:tasks.rados.rados.0.smithi026.stdout:2536: writing smithi02683976-48 from 7962624 to 8380416 tid 2 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2536: writing smithi02683976-48 from 8380416 to 8765440 tid 3 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2537: copy_from oid 26 from oid 34 current snap is 245 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2533: finishing write tid 2 to smithi02683976-6 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2533: finishing write tid 3 to smithi02683976-6 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2533: finishing write tid 4 to smithi02683976-6 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:2533: finishing write tid 5 to smithi02683976-6 2024-09-13T14:27:38.004 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1260 (ObjNum 794 snap 245 seq_num 794) dirty exists 2024-09-13T14:27:38.005 INFO:tasks.rados.rados.0.smithi026.stdout:2533: left oid 6 (ObjNum 794 snap 245 seq_num 794) 2024-09-13T14:27:38.005 INFO:tasks.rados.rados.0.smithi026.stdout:2533: done (4 left) 2024-09-13T14:27:38.005 INFO:tasks.rados.rados.0.smithi026.stdout:2534: done (3 left) 2024-09-13T14:27:38.005 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 245 2024-09-13T14:27:38.005 INFO:tasks.rados.rados.0.smithi026.stdout:2535: expect (ObjNum 768 snap 235 seq_num 768) 2024-09-13T14:27:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:37 smithi145 ceph-mon[89145]: pgmap v437: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-13T14:27:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:37 smithi145 ceph-mon[89145]: osdmap e559: 8 total, 8 up, 8 in 2024-09-13T14:27:38.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:37 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:37 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:38.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:37 smithi145 ceph-mon[89145]: Upgrade: osd.7 is safe to restart 2024-09-13T14:27:38.351 INFO:tasks.rados.rados.0.smithi026.stdout:2536: finishing write tid 1 to smithi02683976-48 2024-09-13T14:27:38.351 INFO:tasks.rados.rados.0.smithi026.stdout:2536: finishing write tid 2 to smithi02683976-48 2024-09-13T14:27:38.351 INFO:tasks.rados.rados.0.smithi026.stdout:2536: finishing write tid 3 to smithi02683976-48 2024-09-13T14:27:38.351 INFO:tasks.rados.rados.0.smithi026.stdout:2536: finishing write tid 4 to smithi02683976-48 2024-09-13T14:27:38.351 INFO:tasks.rados.rados.0.smithi026.stdout:2536: finishing write tid 5 to smithi02683976-48 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1123 (ObjNum 795 snap 245 seq_num 795) dirty exists 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:2536: left oid 48 (ObjNum 795 snap 245 seq_num 795) 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:2537: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:2537: finishing copy_from to smithi02683976-26 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 844 (ObjNum 793 snap 245 seq_num 793) dirty exists 2024-09-13T14:27:38.352 INFO:tasks.rados.rados.0.smithi026.stdout:2538: seq_num 796 ranges {0=540672,540672=704512,1245184=98304} 2024-09-13T14:27:38.354 INFO:tasks.rados.rados.0.smithi026.stdout:2538: writing smithi02683976-25 from 0 to 540672 tid 1 2024-09-13T14:27:38.357 INFO:tasks.rados.rados.0.smithi026.stdout:2538: writing smithi02683976-25 from 540672 to 1245184 tid 2 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2538: writing smithi02683976-25 from 1245184 to 1343488 tid 3 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2535: done (3 left) 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2536: done (2 left) 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2537: done (1 left) 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2539: rollback oid 13 current snap is 245 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 241 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2538: finishing write tid 1 to smithi02683976-25 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 245 2024-09-13T14:27:38.359 INFO:tasks.rados.rados.0.smithi026.stdout:2540: seq_num 797 ranges {4497408=557056,5054464=589824,5644288=212992} 2024-09-13T14:27:38.386 INFO:tasks.rados.rados.0.smithi026.stdout:2540: writing smithi02683976-27 from 4497408 to 5054464 tid 1 2024-09-13T14:27:38.390 INFO:tasks.rados.rados.0.smithi026.stdout:2540: writing smithi02683976-27 from 5054464 to 5644288 tid 2 2024-09-13T14:27:38.391 INFO:tasks.rados.rados.0.smithi026.stdout:2540: writing smithi02683976-27 from 5644288 to 5857280 tid 3 2024-09-13T14:27:38.391 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 245 2024-09-13T14:27:38.391 INFO:tasks.rados.rados.0.smithi026.stdout:2538: finishing write tid 2 to smithi02683976-25 2024-09-13T14:27:38.392 INFO:tasks.rados.rados.0.smithi026.stdout:2541: seq_num 798 ranges {0=573440,573440=466944,1040384=335872} 2024-09-13T14:27:38.395 INFO:tasks.rados.rados.0.smithi026.stdout:2541: writing smithi02683976-18 from 0 to 573440 tid 1 2024-09-13T14:27:38.398 INFO:tasks.rados.rados.0.smithi026.stdout:2541: writing smithi02683976-18 from 573440 to 1040384 tid 2 2024-09-13T14:27:38.400 INFO:tasks.rados.rados.0.smithi026.stdout:2541: writing smithi02683976-18 from 1040384 to 1376256 tid 3 2024-09-13T14:27:38.400 INFO:tasks.rados.rados.0.smithi026.stdout:2542: copy_from oid 43 from oid 10 current snap is 245 2024-09-13T14:27:38.400 INFO:tasks.rados.rados.0.smithi026.stdout:2538: finishing write tid 3 to smithi02683976-25 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2538: finishing write tid 4 to smithi02683976-25 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2539: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2539: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2539: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1183 (ObjNum 766 snap 231 seq_num 766) dirty exists 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2538: finishing write tid 5 to smithi02683976-25 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1040 (ObjNum 796 snap 245 seq_num 796) dirty exists 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2538: left oid 25 (ObjNum 796 snap 245 seq_num 796) 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2540: finishing write tid 1 to smithi02683976-27 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2540: finishing write tid 2 to smithi02683976-27 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2540: finishing write tid 3 to smithi02683976-27 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2540: finishing write tid 4 to smithi02683976-27 2024-09-13T14:27:38.401 INFO:tasks.rados.rados.0.smithi026.stdout:2540: finishing write tid 5 to smithi02683976-27 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1346 (ObjNum 797 snap 245 seq_num 797) dirty exists 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2540: left oid 27 (ObjNum 797 snap 245 seq_num 797) 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2541: finishing write tid 1 to smithi02683976-18 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2538: done (4 left) 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2539: done (3 left) 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2540: done (2 left) 2024-09-13T14:27:38.402 INFO:tasks.rados.rados.0.smithi026.stdout:2543: snap_remove snap 243 2024-09-13T14:27:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[97022]: pgmap v437: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-13T14:27:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[97022]: osdmap e559: 8 total, 8 up, 8 in 2024-09-13T14:27:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[97022]: Upgrade: osd.7 is safe to restart 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[99461]: pgmap v437: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[99461]: osdmap e559: 8 total, 8 up, 8 in 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T14:27:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:37 smithi026 ceph-mon[99461]: Upgrade: osd.7 is safe to restart 2024-09-13T14:27:38.959 INFO:tasks.rados.rados.0.smithi026.stdout:2543: done (2 left) 2024-09-13T14:27:38.959 INFO:tasks.rados.rados.0.smithi026.stdout:2544: read oid 33 snap -1 2024-09-13T14:27:38.959 INFO:tasks.rados.rados.0.smithi026.stdout:2544: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2545: setattr oid 14 current snap is 245 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2541: finishing write tid 2 to smithi02683976-18 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2542: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2541: finishing write tid 3 to smithi02683976-18 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2542: finishing copy_from to smithi02683976-43 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1157 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2541: finishing write tid 4 to smithi02683976-18 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2541: finishing write tid 5 to smithi02683976-18 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1157 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:27:38.960 INFO:tasks.rados.rados.0.smithi026.stdout:2541: left oid 18 (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:27:38.962 INFO:tasks.rados.rados.0.smithi026.stdout:2541: done (3 left) 2024-09-13T14:27:38.962 INFO:tasks.rados.rados.0.smithi026.stdout:2542: done (2 left) 2024-09-13T14:27:38.962 INFO:tasks.rados.rados.0.smithi026.stdout:2546: snap_create 2024-09-13T14:27:38.968 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1158 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:27:38.990 INFO:tasks.rados.rados.0.smithi026.stdout:2544: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:27:39.960 INFO:tasks.rados.rados.0.smithi026.stdout:2544: done (2 left) 2024-09-13T14:27:39.960 INFO:tasks.rados.rados.0.smithi026.stdout:2545: done (1 left) 2024-09-13T14:27:39.961 INFO:tasks.rados.rados.0.smithi026.stdout:2546: done (0 left) 2024-09-13T14:27:39.961 INFO:tasks.rados.rados.0.smithi026.stdout:2547: snap_create 2024-09-13T14:27:40.130 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: Upgrade: Updating osd.7 2024-09-13T14:27:40.130 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: pgmap v439: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.9 MiB/s wr, 11 op/s 2024-09-13T14:27:40.130 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:40.131 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:27:40.131 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:40.131 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: Deploying daemon osd.7 on smithi145 2024-09-13T14:27:40.131 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:39 smithi145 ceph-mon[89145]: osdmap e560: 8 total, 8 up, 8 in 2024-09-13T14:27:40.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: Upgrade: Updating osd.7 2024-09-13T14:27:40.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: pgmap v439: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.9 MiB/s wr, 11 op/s 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: Deploying daemon osd.7 on smithi145 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[97022]: osdmap e560: 8 total, 8 up, 8 in 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: Upgrade: Updating osd.7 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: pgmap v439: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.9 MiB/s wr, 11 op/s 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:40.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: Deploying daemon osd.7 on smithi145 2024-09-13T14:27:40.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:39 smithi026 ceph-mon[99461]: osdmap e560: 8 total, 8 up, 8 in 2024-09-13T14:27:40.397 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:40.964 INFO:tasks.rados.rados.0.smithi026.stdout:2547: done (0 left) 2024-09-13T14:27:40.965 INFO:tasks.rados.rados.0.smithi026.stdout:2548: snap_remove snap 244 2024-09-13T14:27:41.278 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:40 smithi145 ceph-mon[89145]: osdmap e561: 8 total, 8 up, 8 in 2024-09-13T14:27:41.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:40 smithi026 ceph-mon[99461]: osdmap e561: 8 total, 8 up, 8 in 2024-09-13T14:27:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:40 smithi026 ceph-mon[97022]: osdmap e561: 8 total, 8 up, 8 in 2024-09-13T14:27:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:41] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.51.0" 2024-09-13T14:27:41.964 INFO:tasks.rados.rados.0.smithi026.stdout:2548: done (0 left) 2024-09-13T14:27:41.964 INFO:tasks.rados.rados.0.smithi026.stdout:2549: rmattr oid 25 current snap is 247 2024-09-13T14:27:41.964 INFO:tasks.rados.rados.0.smithi026.stdout:2550: read oid 26 snap -1 2024-09-13T14:27:41.965 INFO:tasks.rados.rados.0.smithi026.stdout:2550: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:27:41.965 INFO:tasks.rados.rados.0.smithi026.stdout:2551: delete oid 30 current snap is 247 2024-09-13T14:27:41.969 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 0 (ObjNum 796 snap 245 seq_num 796) dirty exists 2024-09-13T14:27:41.970 INFO:tasks.rados.rados.0.smithi026.stdout:2549: done (2 left) 2024-09-13T14:27:41.970 INFO:tasks.rados.rados.0.smithi026.stdout:2551: done (1 left) 2024-09-13T14:27:41.970 INFO:tasks.rados.rados.0.smithi026.stdout:2552: rollback oid 22 current snap is 247 2024-09-13T14:27:41.970 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 238 2024-09-13T14:27:41.970 INFO:tasks.rados.rados.0.smithi026.stdout:2553: snap_remove snap 225 2024-09-13T14:27:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[99461]: pgmap v442: 121 pgs: 121 active+clean; 234 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 14 op/s 2024-09-13T14:27:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[99461]: osdmap e562: 8 total, 8 up, 8 in 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[97022]: pgmap v442: 121 pgs: 121 active+clean; 234 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 14 op/s 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[97022]: osdmap e562: 8 total, 8 up, 8 in 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:41 smithi145 ceph-mon[89145]: pgmap v442: 121 pgs: 121 active+clean; 234 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 14 op/s 2024-09-13T14:27:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:41 smithi145 ceph-mon[89145]: osdmap e562: 8 total, 8 up, 8 in 2024-09-13T14:27:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:42.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.456 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:27:42.001+0000 7f3580367640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 155 after trimming was started 2024-09-13T14:27:42.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:41 smithi145 systemd[1]: Stopping Ceph osd.7 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:27:42.457 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[54871]: 2024-09-13T14:27:42.018+0000 7fcae633f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:27:42.457 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[54871]: 2024-09-13T14:27:42.018+0000 7fcae633f640 -1 osd.7 563 *** Got signal Terminated *** 2024-09-13T14:27:42.457 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[54871]: 2024-09-13T14:27:42.018+0000 7fcae633f640 -1 osd.7 563 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:27:42.702 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:27:42.981 INFO:tasks.rados.rados.0.smithi026.stdout:2553: done (2 left) 2024-09-13T14:27:42.981 INFO:tasks.rados.rados.0.smithi026.stdout:2554: copy_from oid 37 from oid 45 current snap is 247 2024-09-13T14:27:42.981 INFO:tasks.rados.rados.0.smithi026.stdout:2555: read oid 10 snap -1 2024-09-13T14:27:42.981 INFO:tasks.rados.rados.0.smithi026.stdout:2555: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:42.981 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 247 2024-09-13T14:27:42.982 INFO:tasks.rados.rados.0.smithi026.stdout:2556: seq_num 800 ranges {0=606208,606208=688128,1294336=114688} 2024-09-13T14:27:42.985 INFO:tasks.rados.rados.0.smithi026.stdout:2556: writing smithi02683976-49 from 0 to 606208 tid 1 2024-09-13T14:27:42.991 INFO:tasks.rados.rados.0.smithi026.stdout:2556: writing smithi02683976-49 from 606208 to 1294336 tid 2 2024-09-13T14:27:42.992 INFO:tasks.rados.rados.0.smithi026.stdout:2556: writing smithi02683976-49 from 1294336 to 1409024 tid 3 2024-09-13T14:27:42.999 INFO:tasks.rados.rados.0.smithi026.stdout:2552: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:27:42.999 INFO:tasks.rados.rados.0.smithi026.stdout:2552: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:27:42.999 INFO:tasks.rados.rados.0.smithi026.stdout:2552: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:27:42.999 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1126 (ObjNum 691 snap 195 seq_num 691) dirty exists 2024-09-13T14:27:42.999 INFO:tasks.rados.rados.0.smithi026.stdout:2550: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 2m ago 11m 18.1M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (6m) 25s ago 10m 85.9M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (12m) 2m ago 12m 15.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 18416f2a0cbe 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (12m) 25s ago 12m 23.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (5m) 25s ago 16m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (5m) 2m ago 17m 523M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:27:43.307 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 2m ago 17m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (4m) 25s ago 16m 49.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 2m ago 16m 46.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 2m ago 10m 11.6M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (5m) 25s ago 10m 10.5M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 2m ago 15m 152M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (3m) 2m ago 15m 144M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 2m ago 14m 108M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (2m) 2m ago 14m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (98s) 25s ago 14m 121M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (64s) 25s ago 13m 94.6M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 b9422108344c 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (30s) 25s ago 13m 11.1M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 304ffd9a14d8 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (12m) 25s ago 12m 170M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 325e262959d1 2024-09-13T14:27:43.308 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (5m) 25s ago 11m 49.8M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:27:43.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:42 smithi026 ceph-mon[99461]: osdmap e563: 8 total, 8 up, 8 in 2024-09-13T14:27:43.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:42 smithi026 ceph-mon[99461]: osd.7 marked itself down and dead 2024-09-13T14:27:43.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:42 smithi026 ceph-mon[97022]: osdmap e563: 8 total, 8 up, 8 in 2024-09-13T14:27:43.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:42 smithi026 ceph-mon[97022]: osd.7 marked itself down and dead 2024-09-13T14:27:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-mon[89145]: osdmap e563: 8 total, 8 up, 8 in 2024-09-13T14:27:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:42 smithi145 ceph-mon[89145]: osd.7 marked itself down and dead 2024-09-13T14:27:43.657 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:27:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2, 2024-09-13T14:27:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 12 2024-09-13T14:27:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:27:43.660 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:27:43.809 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:43 smithi145 podman[109977]: 2024-09-13 14:27:43.53211129 +0000 UTC m=+1.761934589 container died 325e262959d1735e978c834a57f5d2926a14b6bfdc3371a704750b7c18ffc785 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T14:27:43.986 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "12/20 daemons upgraded", 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:27:43.987 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:27:44.192 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:43 smithi145 podman[109977]: 2024-09-13 14:27:43.907169268 +0000 UTC m=+2.136992566 container cleanup 325e262959d1735e978c834a57f5d2926a14b6bfdc3371a704750b7c18ffc785 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T14:27:44.193 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:43 smithi145 bash[109977]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: pgmap v445: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 28 op/s 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.193 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4120938282' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:44.194 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:43 smithi145 ceph-mon[89145]: osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:27:44.206 INFO:tasks.rados.rados.0.smithi026.stdout:2555: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2554: finishing copy_from to smithi02683976-37 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2554: got expected ENOENT (src dne) 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2554: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2556: finishing write tid 1 to smithi02683976-49 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2556: finishing write tid 2 to smithi02683976-49 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2556: finishing write tid 3 to smithi02683976-49 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2556: finishing write tid 4 to smithi02683976-49 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:2556: finishing write tid 5 to smithi02683976-49 2024-09-13T14:27:44.208 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1162 (ObjNum 800 snap 247 seq_num 800) dirty exists 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2556: left oid 49 (ObjNum 800 snap 247 seq_num 800) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2550: done (4 left) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2552: done (3 left) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2554: done (2 left) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2555: done (1 left) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2556: done (0 left) 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2557: rollback oid 24 current snap is 247 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 241 2024-09-13T14:27:44.209 INFO:tasks.rados.rados.0.smithi026.stdout:2558: snap_remove snap 240 2024-09-13T14:27:44.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: pgmap v445: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 28 op/s 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4120938282' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[99461]: osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: pgmap v445: 121 pgs: 121 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.6 MiB/s wr, 28 op/s 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4120938282' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:44.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:43 smithi026 ceph-mon[97022]: osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:27:44.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[109997]: 2024-09-13 14:27:44.192208241 +0000 UTC m=+0.658568407 container remove 325e262959d1735e978c834a57f5d2926a14b6bfdc3371a704750b7c18ffc785 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T14:27:44.837 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.405011919 +0000 UTC m=+0.016018606 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:44.837 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.509701835 +0000 UTC m=+0.120708552 container create 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:27:44.837 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.746696486 +0000 UTC m=+0.357703163 container init 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T14:27:44.837 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.751998035 +0000 UTC m=+0.363004710 container start 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:27:44.983 INFO:tasks.rados.rados.0.smithi026.stdout:2558: done (1 left) 2024-09-13T14:27:44.984 INFO:tasks.rados.rados.0.smithi026.stdout:2559: snap_create 2024-09-13T14:27:44.984 INFO:tasks.rados.rados.0.smithi026.stdout:2557: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:27:44.984 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-13T14:27:45.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:44 smithi145 ceph-mon[89145]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:45.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.837030681 +0000 UTC m=+0.448037357 container attach 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:27:45.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:44 smithi145 podman[110093]: 2024-09-13 14:27:44.939680748 +0000 UTC m=+0.550687434 container died 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:27:45.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:44 smithi026 ceph-mon[97022]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:45.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:44 smithi026 ceph-mon[99461]: from='client.44399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:45.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:45 smithi145 podman[110136]: 2024-09-13 14:27:45.590870595 +0000 UTC m=+0.644978812 container remove 64a46fa126cbaf1864aef323861d59618397b12fd077f8c1f0b1208a54fab710 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:27:45.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:45 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T14:27:45.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:45 smithi145 systemd[1]: Stopped Ceph osd.7 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:27:45.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:45 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.7.service: Consumed 15.662s CPU time. 2024-09-13T14:27:45.956 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:45 smithi145 systemd[1]: Starting Ceph osd.7 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:27:45.987 INFO:tasks.rados.rados.0.smithi026.stdout:2557: done (1 left) 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:2559: done (0 left) 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:2560: copy_from oid 22 from oid 29 current snap is 248 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:2561: read oid 10 snap -1 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:2561: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 248 2024-09-13T14:27:45.988 INFO:tasks.rados.rados.0.smithi026.stdout:2562: seq_num 801 ranges {0=622592,622592=573440,1196032=229376} 2024-09-13T14:27:45.991 INFO:tasks.rados.rados.0.smithi026.stdout:2562: writing smithi02683976-47 from 0 to 622592 tid 1 2024-09-13T14:27:45.995 INFO:tasks.rados.rados.0.smithi026.stdout:2562: writing smithi02683976-47 from 622592 to 1196032 tid 2 2024-09-13T14:27:45.997 INFO:tasks.rados.rados.0.smithi026.stdout:2562: writing smithi02683976-47 from 1196032 to 1425408 tid 3 2024-09-13T14:27:45.997 INFO:tasks.rados.rados.0.smithi026.stdout:2563: rollback oid 21 current snap is 248 2024-09-13T14:27:45.997 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 231 2024-09-13T14:27:45.997 INFO:tasks.rados.rados.0.smithi026.stdout:2561: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:45.999 INFO:tasks.rados.rados.0.smithi026.stdout:2562: finishing write tid 1 to smithi02683976-47 2024-09-13T14:27:45.999 INFO:tasks.rados.rados.0.smithi026.stdout:2561: done (3 left) 2024-09-13T14:27:45.999 INFO:tasks.rados.rados.0.smithi026.stdout:2564: setattr oid 11 current snap is 248 2024-09-13T14:27:45.999 INFO:tasks.rados.rados.0.smithi026.stdout:2562: finishing write tid 2 to smithi02683976-47 2024-09-13T14:27:45.999 INFO:tasks.rados.rados.0.smithi026.stdout:2562: finishing write tid 3 to smithi02683976-47 2024-09-13T14:27:46.000 INFO:tasks.rados.rados.0.smithi026.stdout:2562: finishing write tid 4 to smithi02683976-47 2024-09-13T14:27:46.000 INFO:tasks.rados.rados.0.smithi026.stdout:2560: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:27:46.001 INFO:tasks.rados.rados.0.smithi026.stdout:2560: finishing copy_from to smithi02683976-22 2024-09-13T14:27:46.001 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1179 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:27:46.001 INFO:tasks.rados.rados.0.smithi026.stdout:2562: finishing write tid 5 to smithi02683976-47 2024-09-13T14:27:46.001 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1127 (ObjNum 801 snap 248 seq_num 801) dirty exists 2024-09-13T14:27:46.001 INFO:tasks.rados.rados.0.smithi026.stdout:2562: left oid 47 (ObjNum 801 snap 248 seq_num 801) 2024-09-13T14:27:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2560: done (3 left) 2024-09-13T14:27:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2562: done (2 left) 2024-09-13T14:27:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 248 2024-09-13T14:27:46.002 INFO:tasks.rados.rados.0.smithi026.stdout:2565: seq_num 802 ranges {5636096=638976,6275072=458752,6733824=344064} 2024-09-13T14:27:46.036 INFO:tasks.rados.rados.0.smithi026.stdout:2565: writing smithi02683976-31 from 5636096 to 6275072 tid 1 2024-09-13T14:27:46.038 INFO:tasks.rados.rados.0.smithi026.stdout:2565: writing smithi02683976-31 from 6275072 to 6733824 tid 2 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:2565: writing smithi02683976-31 from 6733824 to 7077888 tid 3 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:2566: delete oid 13 current snap is 248 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:2563: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:2563: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:2563: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1268 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:46.041 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1269 (ObjNum 129 snap 0 seq_num 518808248) dirty exists 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2563: done (3 left) 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2564: done (2 left) 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2566: done (1 left) 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2567: setattr oid 23 current snap is 248 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2565: finishing write tid 1 to smithi02683976-31 2024-09-13T14:27:46.044 INFO:tasks.rados.rados.0.smithi026.stdout:2565: finishing write tid 2 to smithi02683976-31 2024-09-13T14:27:46.045 INFO:tasks.rados.rados.0.smithi026.stdout:2565: finishing write tid 3 to smithi02683976-31 2024-09-13T14:27:46.045 INFO:tasks.rados.rados.0.smithi026.stdout:2565: finishing write tid 4 to smithi02683976-31 2024-09-13T14:27:46.047 INFO:tasks.rados.rados.0.smithi026.stdout:2568: copy_from oid 3 from oid 8 current snap is 248 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2569: read oid 39 snap -1 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2569: expect (ObjNum 772 snap 239 seq_num 772) 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2570: rollback oid 11 current snap is 248 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 246 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2565: finishing write tid 5 to smithi02683976-31 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1274 (ObjNum 802 snap 248 seq_num 802) dirty exists 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2565: left oid 31 (ObjNum 802 snap 248 seq_num 802) 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2565: done (4 left) 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 248 2024-09-13T14:27:46.048 INFO:tasks.rados.rados.0.smithi026.stdout:2571: seq_num 804 ranges {0=671744,671744=622592,1294336=180224} 2024-09-13T14:27:46.052 INFO:tasks.rados.rados.0.smithi026.stdout:2571: writing smithi02683976-10 from 0 to 671744 tid 1 2024-09-13T14:27:46.057 INFO:tasks.rados.rados.0.smithi026.stdout:2571: writing smithi02683976-10 from 671744 to 1294336 tid 2 2024-09-13T14:27:46.059 INFO:tasks.rados.rados.0.smithi026.stdout:2571: writing smithi02683976-10 from 1294336 to 1474560 tid 3 2024-09-13T14:27:46.059 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 248 2024-09-13T14:27:46.059 INFO:tasks.rados.rados.0.smithi026.stdout:2570: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:27:46.059 INFO:tasks.rados.rados.0.smithi026.stdout:2572: seq_num 805 ranges {0=688128,688128=507904,1196032=294912} 2024-09-13T14:27:46.064 INFO:tasks.rados.rados.0.smithi026.stdout:2572: writing smithi02683976-35 from 0 to 688128 tid 1 2024-09-13T14:27:46.068 INFO:tasks.rados.rados.0.smithi026.stdout:2572: writing smithi02683976-35 from 688128 to 1196032 tid 2 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2572: writing smithi02683976-35 from 1196032 to 1490944 tid 3 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2573: snap_create 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2570: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1276 (ObjNum 129 snap 0 seq_num 518808248) dirty dne 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1172 (ObjNum 712 snap 208 seq_num 712) dirty exists 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2571: finishing write tid 1 to smithi02683976-10 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2571: finishing write tid 2 to smithi02683976-10 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2571: finishing write tid 3 to smithi02683976-10 2024-09-13T14:27:46.071 INFO:tasks.rados.rados.0.smithi026.stdout:2571: finishing write tid 4 to smithi02683976-10 2024-09-13T14:27:46.072 INFO:tasks.rados.rados.0.smithi026.stdout:2571: finishing write tid 5 to smithi02683976-10 2024-09-13T14:27:46.072 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1166 (ObjNum 804 snap 248 seq_num 804) dirty exists 2024-09-13T14:27:46.072 INFO:tasks.rados.rados.0.smithi026.stdout:2571: left oid 10 (ObjNum 804 snap 248 seq_num 804) 2024-09-13T14:27:46.075 INFO:tasks.rados.rados.0.smithi026.stdout:2569: expect (ObjNum 772 snap 239 seq_num 772) 2024-09-13T14:27:46.295 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:45 smithi145 ceph-mon[89145]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 2 peering, 20 stale+active+clean, 98 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 646 KiB/s wr, 14 op/s 2024-09-13T14:27:46.296 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:45 smithi145 ceph-mon[89145]: osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:27:46.296 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:45.933478332 +0000 UTC m=+0.015952429 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:46.296 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:46.042239478 +0000 UTC m=+0.124713568 container create 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T14:27:46.296 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:46.295309802 +0000 UTC m=+0.377783887 container init 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:27:46.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:45 smithi026 ceph-mon[99461]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 2 peering, 20 stale+active+clean, 98 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 646 KiB/s wr, 14 op/s 2024-09-13T14:27:46.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:45 smithi026 ceph-mon[99461]: osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:27:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:45 smithi026 ceph-mon[97022]: pgmap v448: 121 pgs: 1 active+clean+snaptrim, 2 peering, 20 stale+active+clean, 98 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 646 KiB/s wr, 14 op/s 2024-09-13T14:27:46.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:45 smithi026 ceph-mon[97022]: osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2568: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2572: finishing write tid 1 to smithi02683976-35 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2572: finishing write tid 2 to smithi02683976-35 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2572: finishing write tid 3 to smithi02683976-35 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2572: finishing write tid 4 to smithi02683976-35 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:2572: finishing write tid 5 to smithi02683976-35 2024-09-13T14:27:46.541 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1370 (ObjNum 805 snap 248 seq_num 805) dirty exists 2024-09-13T14:27:46.542 INFO:tasks.rados.rados.0.smithi026.stdout:2572: left oid 35 (ObjNum 805 snap 248 seq_num 805) 2024-09-13T14:27:46.542 INFO:tasks.rados.rados.0.smithi026.stdout:2568: finishing copy_from to smithi02683976-3 2024-09-13T14:27:46.542 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 979 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:27:46.705 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:46.300490951 +0000 UTC m=+0.382965038 container start 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:46.386805908 +0000 UTC m=+0.469279999 container attach 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.706 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:27:46.707 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T14:27:46.707 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T14:27:46.707 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-13T14:27:46.707 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-13T14:27:46.992 INFO:tasks.rados.rados.0.smithi026.stdout:2567: done (6 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2568: done (5 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2569: done (4 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2570: done (3 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2571: done (2 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2572: done (1 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2573: done (0 left) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2574: rollback oid 45 current snap is 249 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 248 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2575: read oid 14 snap -1 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2575: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 249 2024-09-13T14:27:46.993 INFO:tasks.rados.rados.0.smithi026.stdout:2576: seq_num 806 ranges {5857280=704512,6561792=442368,7004160=360448} 2024-09-13T14:27:47.028 INFO:tasks.rados.rados.0.smithi026.stdout:2576: writing smithi02683976-27 from 5857280 to 6561792 tid 1 2024-09-13T14:27:47.030 INFO:tasks.rados.rados.0.smithi026.stdout:2576: writing smithi02683976-27 from 6561792 to 7004160 tid 2 2024-09-13T14:27:47.033 INFO:tasks.rados.rados.0.smithi026.stdout:2576: writing smithi02683976-27 from 7004160 to 7364608 tid 3 2024-09-13T14:27:47.033 INFO:tasks.rados.rados.0.smithi026.stdout:2577: snap_create 2024-09-13T14:27:47.033 INFO:tasks.rados.rados.0.smithi026.stdout:2575: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2574: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2576: finishing write tid 1 to smithi02683976-27 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2576: finishing write tid 2 to smithi02683976-27 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2576: finishing write tid 3 to smithi02683976-27 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2576: finishing write tid 4 to smithi02683976-27 2024-09-13T14:27:47.037 INFO:tasks.rados.rados.0.smithi026.stdout:2576: finishing write tid 5 to smithi02683976-27 2024-09-13T14:27:47.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1377 (ObjNum 806 snap 249 seq_num 806) dirty exists 2024-09-13T14:27:47.038 INFO:tasks.rados.rados.0.smithi026.stdout:2576: left oid 27 (ObjNum 806 snap 249 seq_num 806) 2024-09-13T14:27:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-mon[89145]: osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:27:47.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T14:27:47.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T14:27:47.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate[110254]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 bash[110239]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-13T14:27:47.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:46 smithi145 podman[110239]: 2024-09-13 14:27:46.85475144 +0000 UTC m=+0.937225537 container died 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T14:27:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:46 smithi026 ceph-mon[97022]: osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:27:47.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:46 smithi026 ceph-mon[99461]: osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:27:47.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:47 smithi145 podman[110395]: 2024-09-13 14:27:47.799408719 +0000 UTC m=+0.934033124 container remove 1c36022852a2ac88bc6519ef7a1fd708da302c3c20bab78116c4031ac5a4c99e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2574: done (3 left) 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2575: done (2 left) 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2576: done (1 left) 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2577: done (0 left) 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2578: read oid 2 snap -1 2024-09-13T14:27:47.998 INFO:tasks.rados.rados.0.smithi026.stdout:2578: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:27:47.999 INFO:tasks.rados.rados.0.smithi026.stdout:2579: snap_create 2024-09-13T14:27:48.006 INFO:tasks.rados.rados.0.smithi026.stdout:2578: expect (ObjNum 700 snap 200 seq_num 700) 2024-09-13T14:27:48.254 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:47 smithi145 ceph-mon[89145]: pgmap v451: 121 pgs: 29 active+undersized, 3 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 2 peering, 5 stale+active+clean, 9 active+undersized+degraded, 71 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 344 KiB/s wr, 4 op/s; 31/363 objects degraded (8.540%) 2024-09-13T14:27:48.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:47 smithi145 ceph-mon[89145]: Health check failed: Degraded data redundancy: 31/363 objects degraded (8.540%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:48.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:47 smithi145 ceph-mon[89145]: osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:27:48.255 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 podman[110458]: 2024-09-13 14:27:47.908690765 +0000 UTC m=+0.016031959 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:27:48.255 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 podman[110458]: 2024-09-13 14:27:48.017071999 +0000 UTC m=+0.124413193 container create 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:27:48.255 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 podman[110458]: 2024-09-13 14:27:48.227459587 +0000 UTC m=+0.334800781 container init 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T14:27:48.255 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 podman[110458]: 2024-09-13 14:27:48.232125137 +0000 UTC m=+0.339466332 container start 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2) 2024-09-13T14:27:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[97022]: pgmap v451: 121 pgs: 29 active+undersized, 3 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 2 peering, 5 stale+active+clean, 9 active+undersized+degraded, 71 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 344 KiB/s wr, 4 op/s; 31/363 objects degraded (8.540%) 2024-09-13T14:27:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[97022]: Health check failed: Degraded data redundancy: 31/363 objects degraded (8.540%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:48.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[97022]: osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:27:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[99461]: pgmap v451: 121 pgs: 29 active+undersized, 3 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 2 peering, 5 stale+active+clean, 9 active+undersized+degraded, 71 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 344 KiB/s wr, 4 op/s; 31/363 objects degraded (8.540%) 2024-09-13T14:27:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[99461]: Health check failed: Degraded data redundancy: 31/363 objects degraded (8.540%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:47 smithi026 ceph-mon[99461]: osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:27:48.705 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-osd[110497]: -- 172.21.15.145:0/2538353478 <== mon.1 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557a1128ba40 con 0x557a12058000 2024-09-13T14:27:48.705 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 bash[110458]: 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 2024-09-13T14:27:48.705 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:48 smithi145 systemd[1]: Started Ceph osd.7 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:27:48.962 INFO:tasks.rados.rados.0.smithi026.stdout:2578: done (1 left) 2024-09-13T14:27:48.962 INFO:tasks.rados.rados.0.smithi026.stdout:2579: done (0 left) 2024-09-13T14:27:48.962 INFO:tasks.rados.rados.0.smithi026.stdout:2580: snap_create 2024-09-13T14:27:49.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[99461]: osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:27:49.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[99461]: osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[97022]: osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:49.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:48 smithi026 ceph-mon[97022]: osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:27:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-mon[89145]: osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:27:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:48 smithi145 ceph-mon[89145]: osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:27:49.964 INFO:tasks.rados.rados.0.smithi026.stdout:2580: done (0 left) 2024-09-13T14:27:49.964 INFO:tasks.rados.rados.0.smithi026.stdout:2581: copy_from oid 41 from oid 34 current snap is 252 2024-09-13T14:27:49.964 INFO:tasks.rados.rados.0.smithi026.stdout:2582: read oid 14 snap 247 2024-09-13T14:27:49.964 INFO:tasks.rados.rados.0.smithi026.stdout:2582: expect (ObjNum 520131648 snap 21981 seq_num 520015808) 2024-09-13T14:27:49.964 INFO:tasks.rados.rados.0.smithi026.stdout:2583: snap_remove snap 233 2024-09-13T14:27:50.194 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:49 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:27:49.893+0000 7fc7ac2b0640 -1 Falling back to public interface 2024-09-13T14:27:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:50 smithi145 ceph-mon[89145]: pgmap v454: 121 pgs: 39 active+undersized, 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 11 active+undersized+degraded, 67 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 344 KiB/s wr, 1 op/s; 34/351 objects degraded (9.687%) 2024-09-13T14:27:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:50 smithi145 ceph-mon[89145]: osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:27:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:27:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:27:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:50 smithi026 ceph-mon[97022]: pgmap v454: 121 pgs: 39 active+undersized, 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 11 active+undersized+degraded, 67 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 344 KiB/s wr, 1 op/s; 34/351 objects degraded (9.687%) 2024-09-13T14:27:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:50 smithi026 ceph-mon[97022]: osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:27:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:50 smithi026 ceph-mon[99461]: pgmap v454: 121 pgs: 39 active+undersized, 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 11 active+undersized+degraded, 67 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 344 KiB/s wr, 1 op/s; 34/351 objects degraded (9.687%) 2024-09-13T14:27:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:50 smithi026 ceph-mon[99461]: osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:27:50.967 INFO:tasks.rados.rados.0.smithi026.stdout:2583: done (2 left) 2024-09-13T14:27:50.967 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 252 2024-09-13T14:27:50.967 INFO:tasks.rados.rados.0.smithi026.stdout:2584: seq_num 807 ranges {0=720896,720896=729088,1449984=73728} 2024-09-13T14:27:50.971 INFO:tasks.rados.rados.0.smithi026.stdout:2584: writing smithi02683976-13 from 0 to 720896 tid 1 2024-09-13T14:27:50.977 INFO:tasks.rados.rados.0.smithi026.stdout:2584: writing smithi02683976-13 from 720896 to 1449984 tid 2 2024-09-13T14:27:50.978 INFO:tasks.rados.rados.0.smithi026.stdout:2584: writing smithi02683976-13 from 1449984 to 1523712 tid 3 2024-09-13T14:27:50.978 INFO:tasks.rados.rados.0.smithi026.stdout:2585: delete oid 38 current snap is 252 2024-09-13T14:27:50.980 INFO:tasks.rados.rados.0.smithi026.stdout:2582: expect (ObjNum 520131648 snap 21981 seq_num 520015808) 2024-09-13T14:27:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2581: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:27:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2581: finishing copy_from to smithi02683976-41 2024-09-13T14:27:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 844 (ObjNum 793 snap 245 seq_num 793) dirty exists 2024-09-13T14:27:50.983 INFO:tasks.rados.rados.0.smithi026.stdout:2584: finishing write tid 1 to smithi02683976-13 2024-09-13T14:27:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:2581: done (3 left) 2024-09-13T14:27:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:2582: done (2 left) 2024-09-13T14:27:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:2585: done (1 left) 2024-09-13T14:27:50.984 INFO:tasks.rados.rados.0.smithi026.stdout:2586: snap_remove snap 251 2024-09-13T14:27:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:27:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:27:51] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.51.0" 2024-09-13T14:27:51.972 INFO:tasks.rados.rados.0.smithi026.stdout:2584: finishing write tid 2 to smithi02683976-13 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2584: finishing write tid 3 to smithi02683976-13 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2584: finishing write tid 4 to smithi02683976-13 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2584: finishing write tid 5 to smithi02683976-13 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1218 (ObjNum 807 snap 252 seq_num 807) dirty exists 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2584: left oid 13 (ObjNum 807 snap 252 seq_num 807) 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2584: done (1 left) 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2586: done (0 left) 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 252 2024-09-13T14:27:51.973 INFO:tasks.rados.rados.0.smithi026.stdout:2587: seq_num 809 ranges {7438336=753664,8192000=499712,8691712=303104} 2024-09-13T14:27:52.017 INFO:tasks.rados.rados.0.smithi026.stdout:2587: writing smithi02683976-32 from 7438336 to 8192000 tid 1 2024-09-13T14:27:52.020 INFO:tasks.rados.rados.0.smithi026.stdout:2587: writing smithi02683976-32 from 8192000 to 8691712 tid 2 2024-09-13T14:27:52.022 INFO:tasks.rados.rados.0.smithi026.stdout:2587: writing smithi02683976-32 from 8691712 to 8994816 tid 3 2024-09-13T14:27:52.023 INFO:tasks.rados.rados.0.smithi026.stdout:2588: snap_remove snap 250 2024-09-13T14:27:52.354 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:51 smithi145 ceph-mon[89145]: pgmap v457: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 6.4 MiB/s wr, 19 op/s; 36/372 objects degraded (9.677%) 2024-09-13T14:27:52.354 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:51 smithi145 ceph-mon[89145]: osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:27:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:51 smithi026 ceph-mon[97022]: pgmap v457: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 6.4 MiB/s wr, 19 op/s; 36/372 objects degraded (9.677%) 2024-09-13T14:27:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:51 smithi026 ceph-mon[97022]: osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:27:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:51 smithi026 ceph-mon[99461]: pgmap v457: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 6.4 MiB/s wr, 19 op/s; 36/372 objects degraded (9.677%) 2024-09-13T14:27:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:51 smithi026 ceph-mon[99461]: osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:27:52.980 INFO:tasks.rados.rados.0.smithi026.stdout:2587: finishing write tid 1 to smithi02683976-32 2024-09-13T14:27:52.980 INFO:tasks.rados.rados.0.smithi026.stdout:2587: finishing write tid 2 to smithi02683976-32 2024-09-13T14:27:52.980 INFO:tasks.rados.rados.0.smithi026.stdout:2587: finishing write tid 3 to smithi02683976-32 2024-09-13T14:27:52.980 INFO:tasks.rados.rados.0.smithi026.stdout:2587: finishing write tid 4 to smithi02683976-32 2024-09-13T14:27:52.980 INFO:tasks.rados.rados.0.smithi026.stdout:2587: finishing write tid 5 to smithi02683976-32 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1184 (ObjNum 809 snap 252 seq_num 809) dirty exists 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2587: left oid 32 (ObjNum 809 snap 252 seq_num 809) 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2587: done (1 left) 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2588: done (0 left) 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2589: copy_from oid 16 from oid 18 current snap is 252 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2590: read oid 42 snap 242 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2590: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2591: copy_from oid 10 from oid 28 current snap is 252 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2592: rmattr oid 37 current snap is 252 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2592: done (3 left) 2024-09-13T14:27:52.981 INFO:tasks.rados.rados.0.smithi026.stdout:2593: rmattr oid 22 current snap is 252 2024-09-13T14:27:52.982 INFO:tasks.rados.rados.0.smithi026.stdout:2594: snap_create 2024-09-13T14:27:52.982 INFO:tasks.rados.rados.0.smithi026.stdout:2591: finishing copy_from to smithi02683976-10 2024-09-13T14:27:52.982 INFO:tasks.rados.rados.0.smithi026.stdout:2591: got expected ENOENT (src dne) 2024-09-13T14:27:52.982 INFO:tasks.rados.rados.0.smithi026.stdout:2591: finishing copy_from racing read to smithi02683976-10 2024-09-13T14:27:52.987 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1180 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:27:52.992 INFO:tasks.rados.rados.0.smithi026.stdout:2590: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:27:53.072 INFO:tasks.rados.rados.0.smithi026.stdout:2589: finishing copy_from racing read to smithi02683976-16 2024-09-13T14:27:53.072 INFO:tasks.rados.rados.0.smithi026.stdout:2589: finishing copy_from to smithi02683976-16 2024-09-13T14:27:53.072 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1157 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:27:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[97022]: osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:27:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[99461]: osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:27:53.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:52 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:52 smithi145 ceph-mon[89145]: osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:27:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:52 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:52 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:53.873 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:27:53.606+0000 7fc7ac2b0640 -1 osd.7 563 log_to_monitors true 2024-09-13T14:27:53.987 INFO:tasks.rados.rados.0.smithi026.stdout:2589: done (4 left) 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2590: done (3 left) 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2591: done (2 left) 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2593: done (1 left) 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2594: done (0 left) 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2595: read oid 19 snap -1 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2595: expect deleted 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 253 2024-09-13T14:27:53.988 INFO:tasks.rados.rados.0.smithi026.stdout:2596: seq_num 810 ranges {11337728=770048,12107776=778240,12886016=24576} 2024-09-13T14:27:54.054 INFO:tasks.rados.rados.0.smithi026.stdout:2596: writing smithi02683976-41 from 11337728 to 12107776 tid 1 2024-09-13T14:27:54.058 INFO:tasks.rados.rados.0.smithi026.stdout:2596: writing smithi02683976-41 from 12107776 to 12886016 tid 2 2024-09-13T14:27:54.059 INFO:tasks.rados.rados.0.smithi026.stdout:2596: writing smithi02683976-41 from 12886016 to 12910592 tid 3 2024-09-13T14:27:54.059 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 253 2024-09-13T14:27:54.059 INFO:tasks.rados.rados.0.smithi026.stdout:2597: seq_num 811 ranges {5914624=786432,6701056=663552,7364608=139264} 2024-09-13T14:27:54.095 INFO:tasks.rados.rados.0.smithi026.stdout:2597: writing smithi02683976-7 from 5914624 to 6701056 tid 1 2024-09-13T14:27:54.099 INFO:tasks.rados.rados.0.smithi026.stdout:2597: writing smithi02683976-7 from 6701056 to 7364608 tid 2 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2597: writing smithi02683976-7 from 7364608 to 7503872 tid 3 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2598: snap_create 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: finishing write tid 1 to smithi02683976-41 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: finishing write tid 2 to smithi02683976-41 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: finishing write tid 3 to smithi02683976-41 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: finishing write tid 4 to smithi02683976-41 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: finishing write tid 5 to smithi02683976-41 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 848 (ObjNum 810 snap 253 seq_num 810) dirty exists 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2596: left oid 41 (ObjNum 810 snap 253 seq_num 810) 2024-09-13T14:27:54.101 INFO:tasks.rados.rados.0.smithi026.stdout:2597: finishing write tid 1 to smithi02683976-7 2024-09-13T14:27:54.103 INFO:tasks.rados.rados.0.smithi026.stdout:2597: finishing write tid 2 to smithi02683976-7 2024-09-13T14:27:54.103 INFO:tasks.rados.rados.0.smithi026.stdout:2597: finishing write tid 3 to smithi02683976-7 2024-09-13T14:27:54.103 INFO:tasks.rados.rados.0.smithi026.stdout:2597: finishing write tid 4 to smithi02683976-7 2024-09-13T14:27:54.104 INFO:tasks.rados.rados.0.smithi026.stdout:2597: finishing write tid 5 to smithi02683976-7 2024-09-13T14:27:54.104 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1132 (ObjNum 811 snap 253 seq_num 811) dirty exists 2024-09-13T14:27:54.104 INFO:tasks.rados.rados.0.smithi026.stdout:2597: left oid 7 (ObjNum 811 snap 253 seq_num 811) 2024-09-13T14:27:54.134 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:27:54 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:27:54.023+0000 7fc7a405a640 -1 osd.7 563 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:27:54.134 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: pgmap v460: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.7 MiB/s wr, 24 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 36/375 objects degraded (9.600%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.135 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:53 smithi145 ceph-mon[89145]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: pgmap v460: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.7 MiB/s wr, 24 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 36/375 objects degraded (9.600%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[99461]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: pgmap v460: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.7 MiB/s wr, 24 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 36/375 objects degraded (9.600%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:54.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:27:54.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:54.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:53 smithi026 ceph-mon[97022]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2595: done (3 left) 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2596: done (2 left) 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2597: done (1 left) 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2598: done (0 left) 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2599: read oid 29 snap -1 2024-09-13T14:27:54.992 INFO:tasks.rados.rados.0.smithi026.stdout:2599: expect (ObjNum 607 snap 173 seq_num 607) 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:2600: rollback oid 13 current snap is 254 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 245 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:2601: rmattr oid 47 current snap is 254 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:2601: done (2 left) 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 254 2024-09-13T14:27:54.993 INFO:tasks.rados.rados.0.smithi026.stdout:2602: seq_num 812 ranges {0=802816,802816=548864,1351680=253952} 2024-09-13T14:27:54.998 INFO:tasks.rados.rados.0.smithi026.stdout:2602: writing smithi02683976-14 from 0 to 802816 tid 1 2024-09-13T14:27:55.002 INFO:tasks.rados.rados.0.smithi026.stdout:2602: writing smithi02683976-14 from 802816 to 1351680 tid 2 2024-09-13T14:27:55.004 INFO:tasks.rados.rados.0.smithi026.stdout:2602: writing smithi02683976-14 from 1351680 to 1605632 tid 3 2024-09-13T14:27:55.004 INFO:tasks.rados.rados.0.smithi026.stdout:2603: read oid 26 snap -1 2024-09-13T14:27:55.004 INFO:tasks.rados.rados.0.smithi026.stdout:2603: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:27:55.005 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 254 2024-09-13T14:27:55.005 INFO:tasks.rados.rados.0.smithi026.stdout:2604: seq_num 813 ranges {0=417792,417792=434176,851968=442368,1294336=327680} 2024-09-13T14:27:55.007 INFO:tasks.rados.rados.0.smithi026.stdout:2604: writing smithi02683976-40 from 0 to 417792 tid 1 2024-09-13T14:27:55.010 INFO:tasks.rados.rados.0.smithi026.stdout:2604: writing smithi02683976-40 from 417792 to 851968 tid 2 2024-09-13T14:27:55.014 INFO:tasks.rados.rados.0.smithi026.stdout:2604: writing smithi02683976-40 from 851968 to 1294336 tid 3 2024-09-13T14:27:55.016 INFO:tasks.rados.rados.0.smithi026.stdout:2604: writing smithi02683976-40 from 1294336 to 1622016 tid 4 2024-09-13T14:27:55.017 INFO:tasks.rados.rados.0.smithi026.stdout:2605: snap_create 2024-09-13T14:27:55.017 INFO:tasks.rados.rados.0.smithi026.stdout:2599: expect (ObjNum 607 snap 173 seq_num 607) 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2600: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2602: finishing write tid 1 to smithi02683976-14 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2602: finishing write tid 2 to smithi02683976-14 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 1 to smithi02683976-40 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2602: finishing write tid 3 to smithi02683976-14 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2602: finishing write tid 4 to smithi02683976-14 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2600: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2600: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1226 (ObjNum 766 snap 231 seq_num 766) dirty exists 2024-09-13T14:27:55.043 INFO:tasks.rados.rados.0.smithi026.stdout:2602: finishing write tid 5 to smithi02683976-14 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1191 (ObjNum 812 snap 254 seq_num 812) dirty exists 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2602: left oid 14 (ObjNum 812 snap 254 seq_num 812) 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 2 to smithi02683976-40 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 3 to smithi02683976-40 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 4 to smithi02683976-40 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 5 to smithi02683976-40 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: finishing write tid 6 to smithi02683976-40 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1171 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-13T14:27:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:2604: left oid 40 (ObjNum 813 snap 254 seq_num 813) 2024-09-13T14:27:55.063 INFO:tasks.rados.rados.0.smithi026.stdout:2603: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:27:55.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:54 smithi145 ceph-mon[89145]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:27:55.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:54 smithi145 ceph-mon[89145]: osdmap e575: 8 total, 7 up, 8 in 2024-09-13T14:27:55.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:54 smithi145 ceph-mon[89145]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:55.186 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:54 smithi145 ceph-mon[89145]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:55.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[97022]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:27:55.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[97022]: osdmap e575: 8 total, 7 up, 8 in 2024-09-13T14:27:55.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[97022]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:55.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[97022]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:55.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[99461]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:27:55.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[99461]: osdmap e575: 8 total, 7 up, 8 in 2024-09-13T14:27:55.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[99461]: from='osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:55.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:54 smithi026 ceph-mon[99461]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi145", "root=default"]} : dispatch 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: pgmap v463: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 372 KiB/s wr, 2 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764] boot 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:27:56.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:55 smithi145 ceph-mon[89145]: osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:27:56.275 INFO:tasks.rados.rados.0.smithi026.stdout:2599: done (5 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2600: done (4 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2602: done (3 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2603: done (2 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2604: done (1 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2605: done (0 left) 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2606: copy_from oid 27 from oid 18 current snap is 255 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 255 2024-09-13T14:27:56.276 INFO:tasks.rados.rados.0.smithi026.stdout:2607: seq_num 814 ranges {6086656=442368,6529024=770048,7299072=434176} 2024-09-13T14:27:56.311 INFO:tasks.rados.rados.0.smithi026.stdout:2607: writing smithi02683976-13 from 6086656 to 6529024 tid 1 2024-09-13T14:27:56.315 INFO:tasks.rados.rados.0.smithi026.stdout:2607: writing smithi02683976-13 from 6529024 to 7299072 tid 2 2024-09-13T14:27:56.318 INFO:tasks.rados.rados.0.smithi026.stdout:2607: writing smithi02683976-13 from 7299072 to 7733248 tid 3 2024-09-13T14:27:56.318 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 255 2024-09-13T14:27:56.318 INFO:tasks.rados.rados.0.smithi026.stdout:2606: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:27:56.318 INFO:tasks.rados.rados.0.smithi026.stdout:2608: seq_num 815 ranges {1490944=458752,1949696=655360,2605056=409600,3014656=139264} 2024-09-13T14:27:56.329 INFO:tasks.rados.rados.0.smithi026.stdout:2608: writing smithi02683976-35 from 1490944 to 1949696 tid 1 2024-09-13T14:27:56.332 INFO:tasks.rados.rados.0.smithi026.stdout:2608: writing smithi02683976-35 from 1949696 to 2605056 tid 2 2024-09-13T14:27:56.335 INFO:tasks.rados.rados.0.smithi026.stdout:2608: writing smithi02683976-35 from 2605056 to 3014656 tid 3 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2608: writing smithi02683976-35 from 3014656 to 3153920 tid 4 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2609: read oid 3 snap -1 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2609: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2610: delete oid 15 current snap is 255 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2606: finishing copy_from to smithi02683976-27 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1157 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2607: finishing write tid 1 to smithi02683976-13 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2607: finishing write tid 2 to smithi02683976-13 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2607: finishing write tid 3 to smithi02683976-13 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2607: finishing write tid 4 to smithi02683976-13 2024-09-13T14:27:56.337 INFO:tasks.rados.rados.0.smithi026.stdout:2607: finishing write tid 5 to smithi02683976-13 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1231 (ObjNum 814 snap 255 seq_num 814) dirty exists 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2607: left oid 13 (ObjNum 814 snap 255 seq_num 814) 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 1 to smithi02683976-35 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 2 to smithi02683976-35 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2606: done (4 left) 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2607: done (3 left) 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2610: done (2 left) 2024-09-13T14:27:56.338 INFO:tasks.rados.rados.0.smithi026.stdout:2611: snap_create 2024-09-13T14:27:56.339 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 3 to smithi02683976-35 2024-09-13T14:27:56.339 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 4 to smithi02683976-35 2024-09-13T14:27:56.339 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 5 to smithi02683976-35 2024-09-13T14:27:56.346 INFO:tasks.rados.rados.0.smithi026.stdout:2608: finishing write tid 6 to smithi02683976-35 2024-09-13T14:27:56.346 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1395 (ObjNum 815 snap 255 seq_num 815) dirty exists 2024-09-13T14:27:56.346 INFO:tasks.rados.rados.0.smithi026.stdout:2608: left oid 35 (ObjNum 815 snap 255 seq_num 815) 2024-09-13T14:27:56.363 INFO:tasks.rados.rados.0.smithi026.stdout:2609: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: pgmap v463: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 372 KiB/s wr, 2 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764] boot 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[99461]: osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: pgmap v463: 121 pgs: 39 active+undersized, 11 active+undersized+degraded, 71 active+clean; 247 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 372 KiB/s wr, 2 op/s; 36/375 objects degraded (9.600%) 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: osd.7 [v2:172.21.15.145:6824/2839525764,v1:172.21.15.145:6825/2839525764] boot 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:27:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:55 smithi026 ceph-mon[97022]: osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:27:57.169 INFO:tasks.rados.rados.0.smithi026.stdout:2608: done (2 left) 2024-09-13T14:27:57.169 INFO:tasks.rados.rados.0.smithi026.stdout:2609: done (1 left) 2024-09-13T14:27:57.170 INFO:tasks.rados.rados.0.smithi026.stdout:2611: done (0 left) 2024-09-13T14:27:57.170 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 256 2024-09-13T14:27:57.170 INFO:tasks.rados.rados.0.smithi026.stdout:2612: seq_num 817 ranges {8765440=491520,9256960=417792,9674752=425984,10100736=360448} 2024-09-13T14:27:57.219 INFO:tasks.rados.rados.0.smithi026.stdout:2612: writing smithi02683976-48 from 8765440 to 9256960 tid 1 2024-09-13T14:27:57.222 INFO:tasks.rados.rados.0.smithi026.stdout:2612: writing smithi02683976-48 from 9256960 to 9674752 tid 2 2024-09-13T14:27:57.224 INFO:tasks.rados.rados.0.smithi026.stdout:2612: writing smithi02683976-48 from 9674752 to 10100736 tid 3 2024-09-13T14:27:57.227 INFO:tasks.rados.rados.0.smithi026.stdout:2612: writing smithi02683976-48 from 10100736 to 10461184 tid 4 2024-09-13T14:27:57.227 INFO:tasks.rados.rados.0.smithi026.stdout:2613: delete oid 9 current snap is 256 2024-09-13T14:27:57.227 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 1 to smithi02683976-48 2024-09-13T14:27:57.227 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 2 to smithi02683976-48 2024-09-13T14:27:57.227 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 3 to smithi02683976-48 2024-09-13T14:27:57.230 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 4 to smithi02683976-48 2024-09-13T14:27:57.231 INFO:tasks.rados.rados.0.smithi026.stdout:2613: done (1 left) 2024-09-13T14:27:57.231 INFO:tasks.rados.rados.0.smithi026.stdout:2614: read oid 25 snap -1 2024-09-13T14:27:57.231 INFO:tasks.rados.rados.0.smithi026.stdout:2614: expect (ObjNum 796 snap 245 seq_num 796) 2024-09-13T14:27:57.231 INFO:tasks.rados.rados.0.smithi026.stdout:2615: read oid 18 snap -1 2024-09-13T14:27:57.231 INFO:tasks.rados.rados.0.smithi026.stdout:2615: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2616: copy_from oid 45 from oid 1 current snap is 256 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2617: read oid 20 snap -1 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2617: expect deleted 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2618: rollback oid 21 current snap is 256 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 254 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 5 to smithi02683976-48 2024-09-13T14:27:57.232 INFO:tasks.rados.rados.0.smithi026.stdout:2619: snap_create 2024-09-13T14:27:57.233 INFO:tasks.rados.rados.0.smithi026.stdout:2612: finishing write tid 6 to smithi02683976-48 2024-09-13T14:27:57.233 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1132 (ObjNum 817 snap 256 seq_num 817) dirty exists 2024-09-13T14:27:57.233 INFO:tasks.rados.rados.0.smithi026.stdout:2612: left oid 48 (ObjNum 817 snap 256 seq_num 817) 2024-09-13T14:27:57.233 INFO:tasks.rados.rados.0.smithi026.stdout:2616: finishing copy_from to smithi02683976-45 2024-09-13T14:27:57.233 INFO:tasks.rados.rados.0.smithi026.stdout:2616: got expected ENOENT (src dne) 2024-09-13T14:27:57.234 INFO:tasks.rados.rados.0.smithi026.stdout:2616: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:27:57.235 INFO:tasks.rados.rados.0.smithi026.stdout:2618: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:27:57.237 INFO:tasks.rados.rados.0.smithi026.stdout:2615: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:27:57.274 INFO:tasks.rados.rados.0.smithi026.stdout:2618: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:27:57.274 INFO:tasks.rados.rados.0.smithi026.stdout:2618: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:27:57.274 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1294 (ObjNum 573 snap 163 seq_num 573) dirty exists 2024-09-13T14:27:57.275 INFO:tasks.rados.rados.0.smithi026.stdout:2614: expect (ObjNum 796 snap 245 seq_num 796) 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T14:27:57.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T14:27:57.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:57.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T14:27:57.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T14:27:57.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T14:27:57.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T14:27:57.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T14:27:57.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T14:27:57.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T14:27:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:27:57.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T14:27:57.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T14:27:57.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T14:27:57.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T14:27:57.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T14:27:57.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T14:27:58.174 INFO:tasks.rados.rados.0.smithi026.stdout:2612: done (6 left) 2024-09-13T14:27:58.174 INFO:tasks.rados.rados.0.smithi026.stdout:2614: done (5 left) 2024-09-13T14:27:58.174 INFO:tasks.rados.rados.0.smithi026.stdout:2615: done (4 left) 2024-09-13T14:27:58.174 INFO:tasks.rados.rados.0.smithi026.stdout:2616: done (3 left) 2024-09-13T14:27:58.175 INFO:tasks.rados.rados.0.smithi026.stdout:2617: done (2 left) 2024-09-13T14:27:58.175 INFO:tasks.rados.rados.0.smithi026.stdout:2618: done (1 left) 2024-09-13T14:27:58.175 INFO:tasks.rados.rados.0.smithi026.stdout:2619: done (0 left) 2024-09-13T14:27:58.175 INFO:tasks.rados.rados.0.smithi026.stdout:2620: snap_remove snap 253 2024-09-13T14:27:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all osd 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: pgmap v466: 121 pgs: 12 peering, 29 active+undersized, 6 active+undersized+degraded, 74 active+clean; 257 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 19/390 objects degraded (4.872%) 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[97022]: Upgrade: It appears safe to stop mds.a 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all osd 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T14:27:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: pgmap v466: 121 pgs: 12 peering, 29 active+undersized, 6 active+undersized+degraded, 74 active+clean; 257 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 19/390 objects degraded (4.872%) 2024-09-13T14:27:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T14:27:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T14:27:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:27:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-13T14:27:58.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:58 smithi026 ceph-mon[99461]: Upgrade: It appears safe to stop mds.a 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all osd 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: pgmap v466: 121 pgs: 12 peering, 29 active+undersized, 6 active+undersized+degraded, 74 active+clean; 257 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s; 19/390 objects degraded (4.872%) 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:27:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-13T14:27:58.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:58 smithi145 ceph-mon[89145]: Upgrade: It appears safe to stop mds.a 2024-09-13T14:27:59.181 INFO:tasks.rados.rados.0.smithi026.stdout:2620: done (0 left) 2024-09-13T14:27:59.181 INFO:tasks.rados.rados.0.smithi026.stdout:2621: delete oid 47 current snap is 257 2024-09-13T14:27:59.186 INFO:tasks.rados.rados.0.smithi026.stdout:2621: done (0 left) 2024-09-13T14:27:59.186 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 257 2024-09-13T14:27:59.186 INFO:tasks.rados.rados.0.smithi026.stdout:2622: seq_num 820 ranges {1376256=540672,1916928=475136,2392064=598016,2990080=131072} 2024-09-13T14:27:59.197 INFO:tasks.rados.rados.0.smithi026.stdout:2622: writing smithi02683976-27 from 1376256 to 1916928 tid 1 2024-09-13T14:27:59.199 INFO:tasks.rados.rados.0.smithi026.stdout:2622: writing smithi02683976-27 from 1916928 to 2392064 tid 2 2024-09-13T14:27:59.203 INFO:tasks.rados.rados.0.smithi026.stdout:2622: writing smithi02683976-27 from 2392064 to 2990080 tid 3 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2622: writing smithi02683976-27 from 2990080 to 3121152 tid 4 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2623: read oid 18 snap -1 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2623: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 257 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 1 to smithi02683976-27 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 2 to smithi02683976-27 2024-09-13T14:27:59.205 INFO:tasks.rados.rados.0.smithi026.stdout:2624: seq_num 821 ranges {0=557056,557056=761856,1318912=442368} 2024-09-13T14:27:59.208 INFO:tasks.rados.rados.0.smithi026.stdout:2624: writing smithi02683976-43 from 0 to 557056 tid 1 2024-09-13T14:27:59.213 INFO:tasks.rados.rados.0.smithi026.stdout:2624: writing smithi02683976-43 from 557056 to 1318912 tid 2 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2624: writing smithi02683976-43 from 1318912 to 1761280 tid 3 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2625: read oid 8 snap 248 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2625: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2626: snap_create 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 3 to smithi02683976-27 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 4 to smithi02683976-27 2024-09-13T14:27:59.216 INFO:tasks.rados.rados.0.smithi026.stdout:2623: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:27:59.256 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 5 to smithi02683976-27 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2622: finishing write tid 6 to smithi02683976-27 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1405 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2622: left oid 27 (ObjNum 820 snap 257 seq_num 820) 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: finishing write tid 1 to smithi02683976-43 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: finishing write tid 2 to smithi02683976-43 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: finishing write tid 3 to smithi02683976-43 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: finishing write tid 4 to smithi02683976-43 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: finishing write tid 5 to smithi02683976-43 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1238 (ObjNum 821 snap 257 seq_num 821) dirty exists 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2624: left oid 43 (ObjNum 821 snap 257 seq_num 821) 2024-09-13T14:27:59.257 INFO:tasks.rados.rados.0.smithi026.stdout:2625: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:27:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:59 smithi145 ceph-mon[89145]: osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:27:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:27:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:27:59 smithi145 ceph-mon[89145]: Health check update: Degraded data redundancy: 19/390 objects degraded (4.872%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[97022]: osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[97022]: Health check update: Degraded data redundancy: 19/390 objects degraded (4.872%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[99461]: osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:27:59.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:27:59.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:27:59.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:59.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:27:59 smithi026 ceph-mon[99461]: Health check update: Degraded data redundancy: 19/390 objects degraded (4.872%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:28:00.184 INFO:tasks.rados.rados.0.smithi026.stdout:2622: done (4 left) 2024-09-13T14:28:00.185 INFO:tasks.rados.rados.0.smithi026.stdout:2623: done (3 left) 2024-09-13T14:28:00.185 INFO:tasks.rados.rados.0.smithi026.stdout:2624: done (2 left) 2024-09-13T14:28:00.185 INFO:tasks.rados.rados.0.smithi026.stdout:2625: done (1 left) 2024-09-13T14:28:00.185 INFO:tasks.rados.rados.0.smithi026.stdout:2626: done (0 left) 2024-09-13T14:28:00.185 INFO:tasks.rados.rados.0.smithi026.stdout:2627: snap_remove snap 246 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[99461]: pgmap v469: 121 pgs: 12 peering, 23 active+undersized, 5 active+undersized+degraded, 81 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 18/393 objects degraded (4.580%) 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[99461]: Upgrade: Updating mds.a 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[99461]: Deploying daemon mds.a on smithi026 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[99461]: osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[97022]: pgmap v469: 121 pgs: 12 peering, 23 active+undersized, 5 active+undersized+degraded, 81 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 18/393 objects degraded (4.580%) 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[97022]: Upgrade: Updating mds.a 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[97022]: Deploying daemon mds.a on smithi026 2024-09-13T14:28:00.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-mon[97022]: osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:28:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:00 smithi145 ceph-mon[89145]: pgmap v469: 121 pgs: 12 peering, 23 active+undersized, 5 active+undersized+degraded, 81 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 18/393 objects degraded (4.580%) 2024-09-13T14:28:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:00 smithi145 ceph-mon[89145]: Upgrade: Updating mds.a 2024-09-13T14:28:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:00 smithi145 ceph-mon[89145]: Deploying daemon mds.a on smithi026 2024-09-13T14:28:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:00 smithi145 ceph-mon[89145]: osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:28:00.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:00.916 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:00 smithi026 systemd[1]: Stopping Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:01.188 INFO:tasks.rados.rados.0.smithi026.stdout:2627: done (0 left) 2024-09-13T14:28:01.189 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 258 2024-09-13T14:28:01.189 INFO:tasks.rados.rados.0.smithi026.stdout:2628: seq_num 822 ranges {2138112=573440,2711552=696320,3407872=507904} 2024-09-13T14:28:01.190 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[67082]: 2024-09-13T14:28:00.932+0000 7f293a812640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:28:01.190 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:00 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[67082]: 2024-09-13T14:28:00.932+0000 7f293a812640 -1 mds.a *** got signal Terminated *** 2024-09-13T14:28:01.203 INFO:tasks.rados.rados.0.smithi026.stdout:2628: writing smithi02683976-2 from 2138112 to 2711552 tid 1 2024-09-13T14:28:01.208 INFO:tasks.rados.rados.0.smithi026.stdout:2628: writing smithi02683976-2 from 2711552 to 3407872 tid 2 2024-09-13T14:28:01.211 INFO:tasks.rados.rados.0.smithi026.stdout:2628: writing smithi02683976-2 from 3407872 to 3915776 tid 3 2024-09-13T14:28:01.211 INFO:tasks.rados.rados.0.smithi026.stdout:2629: read oid 25 snap -1 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2629: expect (ObjNum 796 snap 245 seq_num 796) 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2630: read oid 48 snap 241 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2630: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2631: setattr oid 39 current snap is 258 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2628: finishing write tid 1 to smithi02683976-2 2024-09-13T14:28:01.212 INFO:tasks.rados.rados.0.smithi026.stdout:2628: finishing write tid 2 to smithi02683976-2 2024-09-13T14:28:01.213 INFO:tasks.rados.rados.0.smithi026.stdout:2632: copy_from oid 22 from oid 50 current snap is 258 2024-09-13T14:28:01.213 INFO:tasks.rados.rados.0.smithi026.stdout:2633: snap_create 2024-09-13T14:28:01.214 INFO:tasks.rados.rados.0.smithi026.stdout:2628: finishing write tid 3 to smithi02683976-2 2024-09-13T14:28:01.214 INFO:tasks.rados.rados.0.smithi026.stdout:2632: finishing copy_from to smithi02683976-22 2024-09-13T14:28:01.214 INFO:tasks.rados.rados.0.smithi026.stdout:2632: got expected ENOENT (src dne) 2024-09-13T14:28:01.215 INFO:tasks.rados.rados.0.smithi026.stdout:2632: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:28:01.217 INFO:tasks.rados.rados.0.smithi026.stdout:2628: finishing write tid 4 to smithi02683976-2 2024-09-13T14:28:01.219 INFO:tasks.rados.rados.0.smithi026.stdout:2629: expect (ObjNum 796 snap 245 seq_num 796) 2024-09-13T14:28:01.258 INFO:tasks.rados.rados.0.smithi026.stdout:2628: finishing write tid 5 to smithi02683976-2 2024-09-13T14:28:01.259 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1044 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-13T14:28:01.259 INFO:tasks.rados.rados.0.smithi026.stdout:2628: left oid 2 (ObjNum 822 snap 258 seq_num 822) 2024-09-13T14:28:01.259 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1045 (ObjNum 772 snap 239 seq_num 772) dirty exists 2024-09-13T14:28:01.259 INFO:tasks.rados.rados.0.smithi026.stdout:2630: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:28:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:01 smithi145 ceph-mon[89145]: osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:28:01.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:01 smithi026 ceph-mon[99461]: osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:28:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:01 smithi026 ceph-mon[97022]: osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:28:01.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:01] "GET /metrics HTTP/1.1" 200 36156 "" "Prometheus/2.51.0" 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2628: done (5 left) 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2629: done (4 left) 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2630: done (3 left) 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2631: done (2 left) 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2632: done (1 left) 2024-09-13T14:28:02.199 INFO:tasks.rados.rados.0.smithi026.stdout:2633: done (0 left) 2024-09-13T14:28:02.200 INFO:tasks.rados.rados.0.smithi026.stdout:2634: rollback oid 44 current snap is 259 2024-09-13T14:28:02.200 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 256 2024-09-13T14:28:02.200 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 259 2024-09-13T14:28:02.200 INFO:tasks.rados.rados.0.smithi026.stdout:2635: seq_num 823 ranges {2383872=589824,2973696=573440,3547136=630784} 2024-09-13T14:28:02.214 INFO:tasks.rados.rados.0.smithi026.stdout:2635: writing smithi02683976-42 from 2383872 to 2973696 tid 1 2024-09-13T14:28:02.218 INFO:tasks.rados.rados.0.smithi026.stdout:2635: writing smithi02683976-42 from 2973696 to 3547136 tid 2 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2635: writing smithi02683976-42 from 3547136 to 4177920 tid 3 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2636: rollback oid 5 current snap is 259 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 5 to 245 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2634: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2635: finishing write tid 1 to smithi02683976-42 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2634: done (2 left) 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 259 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2635: finishing write tid 2 to smithi02683976-42 2024-09-13T14:28:02.223 INFO:tasks.rados.rados.0.smithi026.stdout:2637: seq_num 824 ranges {10051584=606208,10657792=458752,11116544=573440,11689984=172032} 2024-09-13T14:28:02.281 INFO:tasks.rados.rados.0.smithi026.stdout:2637: writing smithi02683976-37 from 10051584 to 10657792 tid 1 2024-09-13T14:28:02.283 INFO:tasks.rados.rados.0.smithi026.stdout:2637: writing smithi02683976-37 from 10657792 to 11116544 tid 2 2024-09-13T14:28:02.287 INFO:tasks.rados.rados.0.smithi026.stdout:2637: writing smithi02683976-37 from 11116544 to 11689984 tid 3 2024-09-13T14:28:02.288 INFO:tasks.rados.rados.0.smithi026.stdout:2637: writing smithi02683976-37 from 11689984 to 11862016 tid 4 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 259 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2635: finishing write tid 3 to smithi02683976-42 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2635: finishing write tid 4 to smithi02683976-42 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2635: finishing write tid 5 to smithi02683976-42 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1031 (ObjNum 823 snap 259 seq_num 823) dirty exists 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2635: left oid 42 (ObjNum 823 snap 259 seq_num 823) 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2636: finishing rollback tid 0 to smithi02683976-5 2024-09-13T14:28:02.289 INFO:tasks.rados.rados.0.smithi026.stdout:2638: seq_num 825 ranges {0=622592,622592=745472,1368064=458752} 2024-09-13T14:28:02.292 INFO:tasks.rados.rados.0.smithi026.stdout:2638: writing smithi02683976-9 from 0 to 622592 tid 1 2024-09-13T14:28:02.296 INFO:tasks.rados.rados.0.smithi026.stdout:2638: writing smithi02683976-9 from 622592 to 1368064 tid 2 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2638: writing smithi02683976-9 from 1368064 to 1826816 tid 3 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2635: done (3 left) 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2639: read oid 43 snap -1 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2639: expect (ObjNum 821 snap 257 seq_num 821) 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2640: rollback oid 33 current snap is 259 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 248 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2636: finishing rollback tid 1 to smithi02683976-5 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2641: read oid 13 snap -1 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2641: expect (ObjNum 814 snap 255 seq_num 814) 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 259 2024-09-13T14:28:02.300 INFO:tasks.rados.rados.0.smithi026.stdout:2636: finishing rollback tid 2 to smithi02683976-5 2024-09-13T14:28:02.301 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1413 (ObjNum 706 snap 207 seq_num 706) dirty exists 2024-09-13T14:28:02.301 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 1 to smithi02683976-37 2024-09-13T14:28:02.301 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 2 to smithi02683976-37 2024-09-13T14:28:02.301 INFO:tasks.rados.rados.0.smithi026.stdout:2642: seq_num 826 ranges {4177920=638976,4816896=630784,5447680=573440} 2024-09-13T14:28:02.326 INFO:tasks.rados.rados.0.smithi026.stdout:2642: writing smithi02683976-42 from 4177920 to 4816896 tid 1 2024-09-13T14:28:02.330 INFO:tasks.rados.rados.0.smithi026.stdout:2642: writing smithi02683976-42 from 4816896 to 5447680 tid 2 2024-09-13T14:28:02.333 INFO:tasks.rados.rados.0.smithi026.stdout:2642: writing smithi02683976-42 from 5447680 to 6021120 tid 3 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:2636: done (6 left) 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 259 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 3 to smithi02683976-37 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 4 to smithi02683976-37 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 5 to smithi02683976-37 2024-09-13T14:28:02.334 INFO:tasks.rados.rados.0.smithi026.stdout:2643: seq_num 827 ranges {0=655360,655360=516096,1171456=688128} 2024-09-13T14:28:02.337 INFO:tasks.rados.rados.0.smithi026.stdout:2643: writing smithi02683976-19 from 0 to 655360 tid 1 2024-09-13T14:28:02.341 INFO:tasks.rados.rados.0.smithi026.stdout:2643: writing smithi02683976-19 from 655360 to 1171456 tid 2 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2643: writing smithi02683976-19 from 1171456 to 1859584 tid 3 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2644: delete oid 50 current snap is 259 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2637: finishing write tid 6 to smithi02683976-37 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1036 (ObjNum 824 snap 259 seq_num 824) dirty exists 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2637: left oid 37 (ObjNum 824 snap 259 seq_num 824) 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2638: finishing write tid 1 to smithi02683976-9 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2638: finishing write tid 2 to smithi02683976-9 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2638: finishing write tid 3 to smithi02683976-9 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2638: finishing write tid 4 to smithi02683976-9 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2640: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:28:02.346 INFO:tasks.rados.rados.0.smithi026.stdout:2638: finishing write tid 5 to smithi02683976-9 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1308 (ObjNum 825 snap 259 seq_num 825) dirty exists 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:2638: left oid 9 (ObjNum 825 snap 259 seq_num 825) 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:2640: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:2640: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1417 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:28:02.347 INFO:tasks.rados.rados.0.smithi026.stdout:2639: expect (ObjNum 821 snap 257 seq_num 821) 2024-09-13T14:28:02.393 INFO:tasks.rados.rados.0.smithi026.stdout:2641: expect (ObjNum 814 snap 255 seq_num 814) 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: pgmap v472: 121 pgs: 121 active+clean; 263 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/393 objects degraded (4.580%), 5 pgs degraded) 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: fsmap cephfs:1 {0=b=up:active} 2024-09-13T14:28:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: pgmap v472: 121 pgs: 121 active+clean; 263 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/393 objects degraded (4.580%), 5 pgs degraded) 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=b=up:active} 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: pgmap v472: 121 pgs: 121 active+clean; 263 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/393 objects degraded (4.580%), 5 pgs degraded) 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=b=up:active} 2024-09-13T14:28:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:2642: finishing write tid 1 to smithi02683976-42 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:2642: finishing write tid 2 to smithi02683976-42 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:2642: finishing write tid 3 to smithi02683976-42 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:2642: finishing write tid 4 to smithi02683976-42 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:2642: finishing write tid 5 to smithi02683976-42 2024-09-13T14:28:02.890 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1040 (ObjNum 826 snap 259 seq_num 826) dirty exists 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2642: left oid 42 (ObjNum 826 snap 259 seq_num 826) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2643: finishing write tid 1 to smithi02683976-19 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2643: finishing write tid 2 to smithi02683976-19 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2637: done (7 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2638: done (6 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2639: done (5 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2640: done (4 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2641: done (3 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2642: done (2 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2644: done (1 left) 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2645: rollback oid 39 current snap is 259 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 254 2024-09-13T14:28:02.891 INFO:tasks.rados.rados.0.smithi026.stdout:2643: finishing write tid 3 to smithi02683976-19 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:2643: finishing write tid 4 to smithi02683976-19 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:2643: finishing write tid 5 to smithi02683976-19 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1248 (ObjNum 827 snap 259 seq_num 827) dirty exists 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:2643: left oid 19 (ObjNum 827 snap 259 seq_num 827) 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:2643: done (1 left) 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 259 2024-09-13T14:28:02.892 INFO:tasks.rados.rados.0.smithi026.stdout:2646: seq_num 829 ranges {0=688128,688128=729088,1417216=475136} 2024-09-13T14:28:02.893 INFO:tasks.rados.rados.0.smithi026.stdout:2646: writing smithi02683976-47 from 0 to 688128 tid 1 2024-09-13T14:28:02.897 INFO:tasks.rados.rados.0.smithi026.stdout:2646: writing smithi02683976-47 from 688128 to 1417216 tid 2 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2646: writing smithi02683976-47 from 1417216 to 1892352 tid 3 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2647: read oid 30 snap -1 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2647: expect deleted 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2648: read oid 46 snap -1 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2648: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:28:02.901 INFO:tasks.rados.rados.0.smithi026.stdout:2649: rollback oid 49 current snap is 259 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 49 to 230 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2645: finishing rollback tid 0 to smithi02683976-39 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2645: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2645: finishing rollback tid 2 to smithi02683976-39 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1048 (ObjNum 772 snap 239 seq_num 772) dirty exists 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2646: finishing write tid 1 to smithi02683976-47 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2645: done (4 left) 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2650: copy_from oid 44 from oid 38 current snap is 259 2024-09-13T14:28:02.902 INFO:tasks.rados.rados.0.smithi026.stdout:2651: delete oid 11 current snap is 259 2024-09-13T14:28:02.903 INFO:tasks.rados.rados.0.smithi026.stdout:2646: finishing write tid 2 to smithi02683976-47 2024-09-13T14:28:02.903 INFO:tasks.rados.rados.0.smithi026.stdout:2646: finishing write tid 3 to smithi02683976-47 2024-09-13T14:28:02.903 INFO:tasks.rados.rados.0.smithi026.stdout:2647: done (5 left) 2024-09-13T14:28:02.903 INFO:tasks.rados.rados.0.smithi026.stdout:2651: done (4 left) 2024-09-13T14:28:02.904 INFO:tasks.rados.rados.0.smithi026.stdout:2652: snap_remove snap 254 2024-09-13T14:28:03.203 INFO:tasks.rados.rados.0.smithi026.stdout:2652: done (4 left) 2024-09-13T14:28:03.203 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 259 2024-09-13T14:28:03.203 INFO:tasks.rados.rados.0.smithi026.stdout:2653: seq_num 831 ranges {1343488=720896,2064384=499712,2564096=704512} 2024-09-13T14:28:03.215 INFO:tasks.rados.rados.0.smithi026.stdout:2653: writing smithi02683976-25 from 1343488 to 2064384 tid 1 2024-09-13T14:28:03.219 INFO:tasks.rados.rados.0.smithi026.stdout:2653: writing smithi02683976-25 from 2064384 to 2564096 tid 2 2024-09-13T14:28:03.224 INFO:tasks.rados.rados.0.smithi026.stdout:2653: writing smithi02683976-25 from 2564096 to 3268608 tid 3 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2654: read oid 14 snap -1 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2654: expect (ObjNum 812 snap 254 seq_num 812) 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2655: rmattr oid 3 current snap is 259 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2646: finishing write tid 4 to smithi02683976-47 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2649: finishing rollback tid 0 to smithi02683976-49 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2649: finishing rollback tid 1 to smithi02683976-49 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2656: read oid 19 snap 252 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2656: expect deleted 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2657: copy_from oid 8 from oid 31 current snap is 259 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2646: finishing write tid 5 to smithi02683976-47 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1137 (ObjNum 829 snap 259 seq_num 829) dirty exists 2024-09-13T14:28:03.225 INFO:tasks.rados.rados.0.smithi026.stdout:2646: left oid 47 (ObjNum 829 snap 259 seq_num 829) 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:2649: finishing rollback tid 2 to smithi02683976-49 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1174 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:2650: finishing copy_from to smithi02683976-44 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:2650: got expected ENOENT (src dne) 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:2650: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:28:03.226 INFO:tasks.rados.rados.0.smithi026.stdout:2648: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:28:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:03 smithi145 ceph-mon[89145]: osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:28:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:03 smithi026 ceph-mon[97022]: osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:28:03.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:03 smithi026 ceph-mon[99461]: osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:28:03.689 INFO:tasks.rados.rados.0.smithi026.stdout:2653: finishing write tid 1 to smithi02683976-25 2024-09-13T14:28:03.689 INFO:tasks.rados.rados.0.smithi026.stdout:2653: finishing write tid 2 to smithi02683976-25 2024-09-13T14:28:03.689 INFO:tasks.rados.rados.0.smithi026.stdout:2653: finishing write tid 3 to smithi02683976-25 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:2653: finishing write tid 4 to smithi02683976-25 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1427 (ObjNum 670 snap 188 seq_num 670) dirty exists 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:2653: finishing write tid 5 to smithi02683976-25 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1052 (ObjNum 831 snap 259 seq_num 831) dirty exists 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:2653: left oid 25 (ObjNum 831 snap 259 seq_num 831) 2024-09-13T14:28:03.690 INFO:tasks.rados.rados.0.smithi026.stdout:2654: expect (ObjNum 812 snap 254 seq_num 812) 2024-09-13T14:28:03.735 INFO:tasks.rados.rados.0.smithi026.stdout:2657: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:28:03.735 INFO:tasks.rados.rados.0.smithi026.stdout:2657: finishing copy_from to smithi02683976-8 2024-09-13T14:28:03.735 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1274 (ObjNum 802 snap 248 seq_num 802) dirty exists 2024-09-13T14:28:03.735 INFO:tasks.rados.rados.0.smithi026.stdout:2646: done (8 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2648: done (7 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2649: done (6 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2650: done (5 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2653: done (4 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2654: done (3 left) 2024-09-13T14:28:03.736 INFO:tasks.rados.rados.0.smithi026.stdout:2655: done (2 left) 2024-09-13T14:28:03.737 INFO:tasks.rados.rados.0.smithi026.stdout:2656: done (1 left) 2024-09-13T14:28:03.737 INFO:tasks.rados.rados.0.smithi026.stdout:2657: done (0 left) 2024-09-13T14:28:03.737 INFO:tasks.rados.rados.0.smithi026.stdout:2658: snap_create 2024-09-13T14:28:03.967 INFO:tasks.rados.rados.0.smithi026.stdout:2658: done (0 left) 2024-09-13T14:28:03.967 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 260 2024-09-13T14:28:03.967 INFO:tasks.rados.rados.0.smithi026.stdout:2659: seq_num 832 ranges {0=737280,737280=786432,1523712=417792} 2024-09-13T14:28:03.970 INFO:tasks.rados.rados.0.smithi026.stdout:2659: writing smithi02683976-12 from 0 to 737280 tid 1 2024-09-13T14:28:03.975 INFO:tasks.rados.rados.0.smithi026.stdout:2659: writing smithi02683976-12 from 737280 to 1523712 tid 2 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2659: writing smithi02683976-12 from 1523712 to 1941504 tid 3 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2660: read oid 33 snap -1 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2660: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2661: read oid 50 snap 258 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2661: expect deleted 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 260 2024-09-13T14:28:03.978 INFO:tasks.rados.rados.0.smithi026.stdout:2659: finishing write tid 1 to smithi02683976-12 2024-09-13T14:28:03.979 INFO:tasks.rados.rados.0.smithi026.stdout:2662: seq_num 833 ranges {0=761856,761856=671744,1433600=532480} 2024-09-13T14:28:03.983 INFO:tasks.rados.rados.0.smithi026.stdout:2662: writing smithi02683976-45 from 0 to 761856 tid 1 2024-09-13T14:28:03.987 INFO:tasks.rados.rados.0.smithi026.stdout:2662: writing smithi02683976-45 from 761856 to 1433600 tid 2 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2662: writing smithi02683976-45 from 1433600 to 1966080 tid 3 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2663: rollback oid 36 current snap is 260 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 241 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2659: finishing write tid 2 to smithi02683976-12 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2659: finishing write tid 3 to smithi02683976-12 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2659: finishing write tid 4 to smithi02683976-12 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2661: done (4 left) 2024-09-13T14:28:03.992 INFO:tasks.rados.rados.0.smithi026.stdout:2664: snap_remove snap 258 2024-09-13T14:28:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:04 smithi145 ceph-mon[89145]: pgmap v475: 121 pgs: 121 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 25 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:04 smithi145 ceph-mon[89145]: osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:28:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:04 smithi145 ceph-mon[89145]: osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:28:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[97022]: pgmap v475: 121 pgs: 121 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 25 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[97022]: osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:28:04.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[97022]: osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:28:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[99461]: pgmap v475: 121 pgs: 121 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 25 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-13T14:28:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[99461]: osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:28:04.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:04 smithi026 ceph-mon[99461]: osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:28:04.969 INFO:tasks.rados.rados.0.smithi026.stdout:2664: done (4 left) 2024-09-13T14:28:04.969 INFO:tasks.rados.rados.0.smithi026.stdout:2665: delete oid 4 current snap is 260 2024-09-13T14:28:04.969 INFO:tasks.rados.rados.0.smithi026.stdout:2662: finishing write tid 1 to smithi02683976-45 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2659: finishing write tid 5 to smithi02683976-12 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1184 (ObjNum 832 snap 260 seq_num 832) dirty exists 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2659: left oid 12 (ObjNum 832 snap 260 seq_num 832) 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2662: finishing write tid 2 to smithi02683976-45 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2662: finishing write tid 3 to smithi02683976-45 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2662: finishing write tid 4 to smithi02683976-45 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2662: finishing write tid 5 to smithi02683976-45 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 986 (ObjNum 833 snap 260 seq_num 833) dirty exists 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2662: left oid 45 (ObjNum 833 snap 260 seq_num 833) 2024-09-13T14:28:04.970 INFO:tasks.rados.rados.0.smithi026.stdout:2660: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:05.252 INFO:tasks.rados.rados.0.smithi026.stdout:2663: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:28:05.253 INFO:tasks.rados.rados.0.smithi026.stdout:2663: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:28:05.253 INFO:tasks.rados.rados.0.smithi026.stdout:2663: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:28:05.253 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1431 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:28:05.253 INFO:tasks.rados.rados.0.smithi026.stdout:2659: done (4 left) 2024-09-13T14:28:05.253 INFO:tasks.rados.rados.0.smithi026.stdout:2660: done (3 left) 2024-09-13T14:28:05.254 INFO:tasks.rados.rados.0.smithi026.stdout:2662: done (2 left) 2024-09-13T14:28:05.254 INFO:tasks.rados.rados.0.smithi026.stdout:2663: done (1 left) 2024-09-13T14:28:05.254 INFO:tasks.rados.rados.0.smithi026.stdout:2665: done (0 left) 2024-09-13T14:28:05.254 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 260 2024-09-13T14:28:05.254 INFO:tasks.rados.rados.0.smithi026.stdout:2666: seq_num 835 ranges {1474560=794624,2269184=442368,2711552=712704,3424256=49152} 2024-09-13T14:28:05.264 INFO:tasks.rados.rados.0.smithi026.stdout:2666: writing smithi02683976-10 from 1474560 to 2269184 tid 1 2024-09-13T14:28:05.267 INFO:tasks.rados.rados.0.smithi026.stdout:2666: writing smithi02683976-10 from 2269184 to 2711552 tid 2 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:2666: writing smithi02683976-10 from 2711552 to 3424256 tid 3 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:2666: writing smithi02683976-10 from 3424256 to 3473408 tid 4 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:2667: read oid 6 snap -1 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:2667: expect (ObjNum 794 snap 245 seq_num 794) 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:2668: rollback oid 37 current snap is 260 2024-09-13T14:28:05.272 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 242 2024-09-13T14:28:05.273 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 1 to smithi02683976-10 2024-09-13T14:28:05.273 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 2 to smithi02683976-10 2024-09-13T14:28:05.273 INFO:tasks.rados.rados.0.smithi026.stdout:2669: read oid 17 snap -1 2024-09-13T14:28:05.273 INFO:tasks.rados.rados.0.smithi026.stdout:2669: expect deleted 2024-09-13T14:28:05.273 INFO:tasks.rados.rados.0.smithi026.stdout:2670: snap_remove snap 238 2024-09-13T14:28:05.383 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:04 smithi026 podman[125339]: 2024-09-13 14:28:04.939006833 +0000 UTC m=+4.135459896 container died 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T14:28:05.665 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 podman[125339]: 2024-09-13 14:28:05.581368741 +0000 UTC m=+4.777821803 container cleanup 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T14:28:05.666 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 bash[125339]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a 2024-09-13T14:28:06.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:05 smithi026 ceph-mon[99461]: pgmap v478: 121 pgs: 121 active+clean; 270 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-13T14:28:06.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:05 smithi026 ceph-mon[99461]: osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:28:06.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:05 smithi026 ceph-mon[97022]: pgmap v478: 121 pgs: 121 active+clean; 270 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-13T14:28:06.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:05 smithi026 ceph-mon[97022]: osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:28:06.125 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 podman[125356]: 2024-09-13 14:28:05.774473487 +0000 UTC m=+0.829763387 container remove 18416f2a0cbe6dce96e8e9c99962b521ba07b70c6d3183f5da028fe272d66fea (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:28:06.125 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.a.service: Deactivated successfully. 2024-09-13T14:28:06.125 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 systemd[1]: Stopped Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:06.125 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:05 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.a.service: Consumed 1.185s CPU time. 2024-09-13T14:28:06.125 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 systemd[1]: Starting Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:06.207 INFO:tasks.rados.rados.0.smithi026.stdout:2670: done (4 left) 2024-09-13T14:28:06.207 INFO:tasks.rados.rados.0.smithi026.stdout:2671: read oid 7 snap -1 2024-09-13T14:28:06.207 INFO:tasks.rados.rados.0.smithi026.stdout:2671: expect (ObjNum 811 snap 253 seq_num 811) 2024-09-13T14:28:06.207 INFO:tasks.rados.rados.0.smithi026.stdout:2672: snap_remove snap 231 2024-09-13T14:28:06.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:05 smithi145 ceph-mon[89145]: pgmap v478: 121 pgs: 121 active+clean; 270 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-13T14:28:06.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:05 smithi145 ceph-mon[89145]: osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:28:06.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:28:06 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:28:06.235+0000 7ff25c8cf640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 14f after trimming was started 2024-09-13T14:28:06.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 podman[125492]: 2024-09-13 14:28:06.257949711 +0000 UTC m=+0.018075274 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:28:06.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 podman[125492]: 2024-09-13 14:28:06.367589961 +0000 UTC m=+0.127715513 container create e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:28:06.915 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 podman[125492]: 2024-09-13 14:28:06.611974513 +0000 UTC m=+0.372100067 container init e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:28:06.915 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 podman[125492]: 2024-09-13 14:28:06.617407856 +0000 UTC m=+0.377533418 container start e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:28:06.916 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: starting mds.a at 2024-09-13T14:28:06.916 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 bash[125492]: e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 2024-09-13T14:28:06.916 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:06 smithi026 systemd[1]: Started Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2672: done (5 left) 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2673: read oid 15 snap -1 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2673: expect deleted 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2674: copy_from oid 24 from oid 19 current snap is 260 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2675: snap_create 2024-09-13T14:28:07.214 INFO:tasks.rados.rados.0.smithi026.stdout:2668: finishing rollback tid 0 to smithi02683976-37 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2668: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2668: finishing rollback tid 2 to smithi02683976-37 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1043 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 3 to smithi02683976-10 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 4 to smithi02683976-10 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 5 to smithi02683976-10 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2666: finishing write tid 6 to smithi02683976-10 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1179 (ObjNum 835 snap 260 seq_num 835) dirty exists 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2666: left oid 10 (ObjNum 835 snap 260 seq_num 835) 2024-09-13T14:28:07.215 INFO:tasks.rados.rados.0.smithi026.stdout:2667: expect (ObjNum 794 snap 245 seq_num 794) 2024-09-13T14:28:07.346 INFO:tasks.rados.rados.0.smithi026.stdout:2671: expect (ObjNum 811 snap 253 seq_num 811) 2024-09-13T14:28:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:07 smithi145 ceph-mon[89145]: osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:28:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:07 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:07.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[99461]: osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:28:07.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:07.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[97022]: osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:28:07.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:07.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:07 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:07.832 INFO:tasks.rados.rados.0.smithi026.stdout:2674: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:28:07.833 INFO:tasks.rados.rados.0.smithi026.stdout:2674: finishing copy_from to smithi02683976-24 2024-09-13T14:28:07.833 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1248 (ObjNum 827 snap 259 seq_num 827) dirty exists 2024-09-13T14:28:08.214 INFO:tasks.rados.rados.0.smithi026.stdout:2666: done (7 left) 2024-09-13T14:28:08.214 INFO:tasks.rados.rados.0.smithi026.stdout:2667: done (6 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2668: done (5 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2669: done (4 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2671: done (3 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2673: done (2 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2674: done (1 left) 2024-09-13T14:28:08.215 INFO:tasks.rados.rados.0.smithi026.stdout:2675: done (0 left) 2024-09-13T14:28:08.216 INFO:tasks.rados.rados.0.smithi026.stdout:2676: rollback oid 6 current snap is 261 2024-09-13T14:28:08.216 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 252 2024-09-13T14:28:08.216 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 261 2024-09-13T14:28:08.216 INFO:tasks.rados.rados.0.smithi026.stdout:2677: seq_num 836 ranges {1826816=409600,2236416=720896,2957312=516096,3473408=368640} 2024-09-13T14:28:08.226 INFO:tasks.rados.rados.0.smithi026.stdout:2677: writing smithi02683976-9 from 1826816 to 2236416 tid 1 2024-09-13T14:28:08.230 INFO:tasks.rados.rados.0.smithi026.stdout:2677: writing smithi02683976-9 from 2236416 to 2957312 tid 2 2024-09-13T14:28:08.233 INFO:tasks.rados.rados.0.smithi026.stdout:2677: writing smithi02683976-9 from 2957312 to 3473408 tid 3 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:2677: writing smithi02683976-9 from 3473408 to 3842048 tid 4 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:2678: copy_from oid 19 from oid 31 current snap is 261 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:2676: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:2676: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:2676: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:28:08.236 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1334 (ObjNum 794 snap 245 seq_num 794) dirty exists 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:2676: done (2 left) 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:2679: rollback oid 34 current snap is 261 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 260 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 1 to smithi02683976-9 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 2 to smithi02683976-9 2024-09-13T14:28:08.237 INFO:tasks.rados.rados.0.smithi026.stdout:2680: delete oid 7 current snap is 261 2024-09-13T14:28:08.239 INFO:tasks.rados.rados.0.smithi026.stdout:2680: done (3 left) 2024-09-13T14:28:08.239 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 3 to smithi02683976-9 2024-09-13T14:28:08.240 INFO:tasks.rados.rados.0.smithi026.stdout:2681: read oid 5 snap -1 2024-09-13T14:28:08.240 INFO:tasks.rados.rados.0.smithi026.stdout:2681: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:28:08.240 INFO:tasks.rados.rados.0.smithi026.stdout:2682: delete oid 7 current snap is 261 2024-09-13T14:28:08.240 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 4 to smithi02683976-9 2024-09-13T14:28:08.240 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 5 to smithi02683976-9 2024-09-13T14:28:08.242 INFO:tasks.rados.rados.0.smithi026.stdout:2679: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:28:08.242 INFO:tasks.rados.rados.0.smithi026.stdout:2677: finishing write tid 6 to smithi02683976-9 2024-09-13T14:28:08.242 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1340 (ObjNum 836 snap 261 seq_num 836) dirty exists 2024-09-13T14:28:08.242 INFO:tasks.rados.rados.0.smithi026.stdout:2677: left oid 9 (ObjNum 836 snap 261 seq_num 836) 2024-09-13T14:28:08.242 INFO:tasks.rados.rados.0.smithi026.stdout:2679: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:28:08.243 INFO:tasks.rados.rados.0.smithi026.stdout:2679: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:28:08.243 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 851 (ObjNum 793 snap 245 seq_num 793) dirty exists 2024-09-13T14:28:08.245 INFO:tasks.rados.rados.0.smithi026.stdout:2677: done (4 left) 2024-09-13T14:28:08.245 INFO:tasks.rados.rados.0.smithi026.stdout:2679: done (3 left) 2024-09-13T14:28:08.245 INFO:tasks.rados.rados.0.smithi026.stdout:2682: done (2 left) 2024-09-13T14:28:08.245 INFO:tasks.rados.rados.0.smithi026.stdout:2683: read oid 45 snap -1 2024-09-13T14:28:08.246 INFO:tasks.rados.rados.0.smithi026.stdout:2683: expect (ObjNum 833 snap 260 seq_num 833) 2024-09-13T14:28:08.246 INFO:tasks.rados.rados.0.smithi026.stdout:2684: copy_from oid 9 from oid 20 current snap is 261 2024-09-13T14:28:08.246 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 261 2024-09-13T14:28:08.246 INFO:tasks.rados.rados.0.smithi026.stdout:2685: seq_num 839 ranges {12910592=458752,13369344=425984,13795328=688128,14483456=491520} 2024-09-13T14:28:08.322 INFO:tasks.rados.rados.0.smithi026.stdout:2685: writing smithi02683976-41 from 12910592 to 13369344 tid 1 2024-09-13T14:28:08.324 INFO:tasks.rados.rados.0.smithi026.stdout:2685: writing smithi02683976-41 from 13369344 to 13795328 tid 2 2024-09-13T14:28:08.328 INFO:tasks.rados.rados.0.smithi026.stdout:2685: writing smithi02683976-41 from 13795328 to 14483456 tid 3 2024-09-13T14:28:08.331 INFO:tasks.rados.rados.0.smithi026.stdout:2685: writing smithi02683976-41 from 14483456 to 14974976 tid 4 2024-09-13T14:28:08.331 INFO:tasks.rados.rados.0.smithi026.stdout:2686: delete oid 24 current snap is 261 2024-09-13T14:28:08.332 INFO:tasks.rados.rados.0.smithi026.stdout:2684: finishing copy_from to smithi02683976-9 2024-09-13T14:28:08.332 INFO:tasks.rados.rados.0.smithi026.stdout:2684: got expected ENOENT (src dne) 2024-09-13T14:28:08.332 INFO:tasks.rados.rados.0.smithi026.stdout:2684: finishing copy_from racing read to smithi02683976-9 2024-09-13T14:28:08.332 INFO:tasks.rados.rados.0.smithi026.stdout:2683: expect (ObjNum 833 snap 260 seq_num 833) 2024-09-13T14:28:08.389 INFO:tasks.rados.rados.0.smithi026.stdout:2678: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:28:08.390 INFO:tasks.rados.rados.0.smithi026.stdout:2681: expect (ObjNum 706 snap 207 seq_num 706) 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[99461]: pgmap v481: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 289 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[99461]: osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:boot 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:08.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[97022]: pgmap v481: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 289 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-09-13T14:28:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[97022]: osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:28:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:boot 2024-09-13T14:28:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:28:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:08 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:08 smithi145 ceph-mon[89145]: pgmap v481: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 289 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-09-13T14:28:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:08 smithi145 ceph-mon[89145]: osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:28:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:08 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:boot 2024-09-13T14:28:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:08 smithi145 ceph-mon[89145]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-13T14:28:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:08 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:28:09.176 INFO:tasks.rados.rados.0.smithi026.stdout:2678: finishing copy_from to smithi02683976-19 2024-09-13T14:28:09.176 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1274 (ObjNum 802 snap 248 seq_num 802) dirty exists 2024-09-13T14:28:09.176 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 1 to smithi02683976-41 2024-09-13T14:28:09.176 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 2 to smithi02683976-41 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2678: done (5 left) 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2681: done (4 left) 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2683: done (3 left) 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2684: done (2 left) 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2686: done (1 left) 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2687: setattr oid 29 current snap is 261 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 3 to smithi02683976-41 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 4 to smithi02683976-41 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 5 to smithi02683976-41 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2685: finishing write tid 6 to smithi02683976-41 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 856 (ObjNum 839 snap 261 seq_num 839) dirty exists 2024-09-13T14:28:09.177 INFO:tasks.rados.rados.0.smithi026.stdout:2685: left oid 41 (ObjNum 839 snap 261 seq_num 839) 2024-09-13T14:28:09.178 INFO:tasks.rados.rados.0.smithi026.stdout:2685: done (1 left) 2024-09-13T14:28:09.178 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 261 2024-09-13T14:28:09.178 INFO:tasks.rados.rados.0.smithi026.stdout:2688: seq_num 841 ranges {7438336=491520,7929856=598016,8527872=704512,9232384=303104} 2024-09-13T14:28:09.222 INFO:tasks.rados.rados.0.smithi026.stdout:2688: writing smithi02683976-3 from 7438336 to 7929856 tid 1 2024-09-13T14:28:09.226 INFO:tasks.rados.rados.0.smithi026.stdout:2688: writing smithi02683976-3 from 7929856 to 8527872 tid 2 2024-09-13T14:28:09.230 INFO:tasks.rados.rados.0.smithi026.stdout:2688: writing smithi02683976-3 from 8527872 to 9232384 tid 3 2024-09-13T14:28:09.232 INFO:tasks.rados.rados.0.smithi026.stdout:2688: writing smithi02683976-3 from 9232384 to 9535488 tid 4 2024-09-13T14:28:09.232 INFO:tasks.rados.rados.0.smithi026.stdout:2689: setattr oid 37 current snap is 261 2024-09-13T14:28:09.232 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1275 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:28:09.232 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 1 to smithi02683976-3 2024-09-13T14:28:09.232 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 2 to smithi02683976-3 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2687: done (2 left) 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2690: rollback oid 17 current snap is 261 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 17 to 252 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2691: read oid 34 snap -1 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2691: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2692: read oid 19 snap -1 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2692: expect (ObjNum 802 snap 248 seq_num 802) 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2693: rollback oid 6 current snap is 261 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 241 2024-09-13T14:28:09.234 INFO:tasks.rados.rados.0.smithi026.stdout:2694: snap_create 2024-09-13T14:28:09.235 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 3 to smithi02683976-3 2024-09-13T14:28:09.236 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 4 to smithi02683976-3 2024-09-13T14:28:09.239 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1044 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:28:09.239 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 5 to smithi02683976-3 2024-09-13T14:28:09.241 INFO:tasks.rados.rados.0.smithi026.stdout:2688: finishing write tid 6 to smithi02683976-3 2024-09-13T14:28:09.241 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1469 (ObjNum 841 snap 261 seq_num 841) dirty exists 2024-09-13T14:28:09.241 INFO:tasks.rados.rados.0.smithi026.stdout:2688: left oid 3 (ObjNum 841 snap 261 seq_num 841) 2024-09-13T14:28:09.242 INFO:tasks.rados.rados.0.smithi026.stdout:2693: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:28:09.242 INFO:tasks.rados.rados.0.smithi026.stdout:2693: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:28:09.243 INFO:tasks.rados.rados.0.smithi026.stdout:2693: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:28:09.243 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1343 (ObjNum 737 snap 218 seq_num 737) dirty exists 2024-09-13T14:28:09.262 INFO:tasks.rados.rados.0.smithi026.stdout:2692: expect (ObjNum 802 snap 248 seq_num 802) 2024-09-13T14:28:09.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:09 smithi026 ceph-mon[99461]: osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:28:09.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:09 smithi026 ceph-mon[99461]: osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:28:09.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:09 smithi026 ceph-mon[97022]: osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:28:09.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:09 smithi026 ceph-mon[97022]: osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:28:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:09 smithi145 ceph-mon[89145]: osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:28:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:09 smithi145 ceph-mon[89145]: osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:28:09.830 INFO:tasks.rados.rados.0.smithi026.stdout:2690: finishing rollback tid 1 to smithi02683976-17 2024-09-13T14:28:09.830 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 840 (ObjNum 593 snap 0 seq_num 0) dirty dne 2024-09-13T14:28:09.830 INFO:tasks.rados.rados.0.smithi026.stdout:2691: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:28:10.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[99461]: pgmap v484: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-13T14:28:10.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[97022]: pgmap v484: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-13T14:28:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:10 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:10 smithi145 ceph-mon[89145]: pgmap v484: 121 pgs: 2 active+clean+snaptrim_wait, 119 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-13T14:28:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:10 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:10.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2688: done (6 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2689: done (5 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2690: done (4 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2691: done (3 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2692: done (2 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2693: done (1 left) 2024-09-13T14:28:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:2694: done (0 left) 2024-09-13T14:28:11.046 INFO:tasks.rados.rados.0.smithi026.stdout:2695: snap_create 2024-09-13T14:28:11.517 INFO:tasks.rados.rados.0.smithi026.stdout:2695: done (0 left) 2024-09-13T14:28:11.517 INFO:tasks.rados.rados.0.smithi026.stdout:2696: read oid 34 snap -1 2024-09-13T14:28:11.517 INFO:tasks.rados.rados.0.smithi026.stdout:2696: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:28:11.518 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 263 2024-09-13T14:28:11.518 INFO:tasks.rados.rados.0.smithi026.stdout:2697: seq_num 842 ranges {7077888=507904,7585792=475136,8060928=516096,8577024=499712,9076736=114688} 2024-09-13T14:28:11.558 INFO:tasks.rados.rados.0.smithi026.stdout:2697: writing smithi02683976-8 from 7077888 to 7585792 tid 1 2024-09-13T14:28:11.561 INFO:tasks.rados.rados.0.smithi026.stdout:2697: writing smithi02683976-8 from 7585792 to 8060928 tid 2 2024-09-13T14:28:11.565 INFO:tasks.rados.rados.0.smithi026.stdout:2697: writing smithi02683976-8 from 8060928 to 8577024 tid 3 2024-09-13T14:28:11.569 INFO:tasks.rados.rados.0.smithi026.stdout:2697: writing smithi02683976-8 from 8577024 to 9076736 tid 4 2024-09-13T14:28:11.570 INFO:tasks.rados.rados.0.smithi026.stdout:2697: writing smithi02683976-8 from 9076736 to 9191424 tid 5 2024-09-13T14:28:11.570 INFO:tasks.rados.rados.0.smithi026.stdout:2698: delete oid 35 current snap is 263 2024-09-13T14:28:11.570 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 1 to smithi02683976-8 2024-09-13T14:28:11.570 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 2 to smithi02683976-8 2024-09-13T14:28:11.570 INFO:tasks.rados.rados.0.smithi026.stdout:2696: expect (ObjNum 793 snap 245 seq_num 793) 2024-09-13T14:28:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:11] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.51.0" 2024-09-13T14:28:11.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[99461]: osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[97022]: osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:11 smithi145 ceph-mon[89145]: osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:28:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:12.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:12 smithi026 ceph-mon[97022]: pgmap v487: 121 pgs: 121 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 4.1 MiB/s wr, 19 op/s 2024-09-13T14:28:12.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:12 smithi026 ceph-mon[97022]: osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:28:12.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:12 smithi026 ceph-mon[99461]: pgmap v487: 121 pgs: 121 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 4.1 MiB/s wr, 19 op/s 2024-09-13T14:28:12.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:12 smithi026 ceph-mon[99461]: osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:28:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:12 smithi145 ceph-mon[89145]: pgmap v487: 121 pgs: 121 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 4.1 MiB/s wr, 19 op/s 2024-09-13T14:28:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:12 smithi145 ceph-mon[89145]: osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 3 to smithi02683976-8 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 4 to smithi02683976-8 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 5 to smithi02683976-8 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 6 to smithi02683976-8 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: finishing write tid 7 to smithi02683976-8 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1280 (ObjNum 842 snap 263 seq_num 842) dirty exists 2024-09-13T14:28:12.772 INFO:tasks.rados.rados.0.smithi026.stdout:2697: left oid 8 (ObjNum 842 snap 263 seq_num 842) 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2696: done (2 left) 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2697: done (1 left) 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2698: done (0 left) 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2699: rollback oid 2 current snap is 263 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 259 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2700: copy_from oid 18 from oid 31 current snap is 263 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2701: read oid 46 snap 257 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2701: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 263 2024-09-13T14:28:12.773 INFO:tasks.rados.rados.0.smithi026.stdout:2702: seq_num 844 ranges {573440=540672,1114112=696320,1810432=475136,2285568=434176} 2024-09-13T14:28:12.778 INFO:tasks.rados.rados.0.smithi026.stdout:2702: writing smithi02683976-29 from 573440 to 1114112 tid 1 2024-09-13T14:28:12.782 INFO:tasks.rados.rados.0.smithi026.stdout:2702: writing smithi02683976-29 from 1114112 to 1810432 tid 2 2024-09-13T14:28:12.785 INFO:tasks.rados.rados.0.smithi026.stdout:2702: writing smithi02683976-29 from 1810432 to 2285568 tid 3 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2702: writing smithi02683976-29 from 2285568 to 2719744 tid 4 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 263 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2699: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2699: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2699: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1252 (ObjNum 822 snap 258 seq_num 822) dirty exists 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 1 to smithi02683976-29 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 2 to smithi02683976-29 2024-09-13T14:28:12.789 INFO:tasks.rados.rados.0.smithi026.stdout:2703: seq_num 845 ranges {1761280=557056,2318336=581632,2899968=688128,3588096=335872} 2024-09-13T14:28:12.803 INFO:tasks.rados.rados.0.smithi026.stdout:2703: writing smithi02683976-43 from 1761280 to 2318336 tid 1 2024-09-13T14:28:12.806 INFO:tasks.rados.rados.0.smithi026.stdout:2703: writing smithi02683976-43 from 2318336 to 2899968 tid 2 2024-09-13T14:28:12.810 INFO:tasks.rados.rados.0.smithi026.stdout:2703: writing smithi02683976-43 from 2899968 to 3588096 tid 3 2024-09-13T14:28:12.812 INFO:tasks.rados.rados.0.smithi026.stdout:2703: writing smithi02683976-43 from 3588096 to 3923968 tid 4 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 3 to smithi02683976-29 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 4 to smithi02683976-29 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 5 to smithi02683976-29 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: finishing write tid 6 to smithi02683976-29 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1280 (ObjNum 844 snap 263 seq_num 844) dirty exists 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: left oid 29 (ObjNum 844 snap 263 seq_num 844) 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 1 to smithi02683976-43 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2700: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 2 to smithi02683976-43 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2699: done (4 left) 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:2702: done (3 left) 2024-09-13T14:28:12.813 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 263 2024-09-13T14:28:12.814 INFO:tasks.rados.rados.0.smithi026.stdout:2704: seq_num 846 ranges {368640=573440,942080=466944,1409024=491520,1900544=548864,2449408=98304} 2024-09-13T14:28:12.818 INFO:tasks.rados.rados.0.smithi026.stdout:2704: writing smithi02683976-36 from 368640 to 942080 tid 1 2024-09-13T14:28:12.821 INFO:tasks.rados.rados.0.smithi026.stdout:2704: writing smithi02683976-36 from 942080 to 1409024 tid 2 2024-09-13T14:28:12.824 INFO:tasks.rados.rados.0.smithi026.stdout:2704: writing smithi02683976-36 from 1409024 to 1900544 tid 3 2024-09-13T14:28:12.828 INFO:tasks.rados.rados.0.smithi026.stdout:2704: writing smithi02683976-36 from 1900544 to 2449408 tid 4 2024-09-13T14:28:12.829 INFO:tasks.rados.rados.0.smithi026.stdout:2704: writing smithi02683976-36 from 2449408 to 2547712 tid 5 2024-09-13T14:28:12.829 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 263 2024-09-13T14:28:12.829 INFO:tasks.rados.rados.0.smithi026.stdout:2700: finishing copy_from to smithi02683976-18 2024-09-13T14:28:12.829 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1274 (ObjNum 802 snap 248 seq_num 802) dirty exists 2024-09-13T14:28:12.829 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 3 to smithi02683976-43 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 4 to smithi02683976-43 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 5 to smithi02683976-43 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2703: finishing write tid 6 to smithi02683976-43 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1285 (ObjNum 845 snap 263 seq_num 845) dirty exists 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2703: left oid 43 (ObjNum 845 snap 263 seq_num 845) 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 1 to smithi02683976-36 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 2 to smithi02683976-36 2024-09-13T14:28:12.830 INFO:tasks.rados.rados.0.smithi026.stdout:2705: seq_num 847 ranges {6119424=589824,6709248=753664,7462912=655360,8118272=196608} 2024-09-13T14:28:12.866 INFO:tasks.rados.rados.0.smithi026.stdout:2705: writing smithi02683976-39 from 6119424 to 6709248 tid 1 2024-09-13T14:28:12.870 INFO:tasks.rados.rados.0.smithi026.stdout:2705: writing smithi02683976-39 from 6709248 to 7462912 tid 2 2024-09-13T14:28:12.874 INFO:tasks.rados.rados.0.smithi026.stdout:2705: writing smithi02683976-39 from 7462912 to 8118272 tid 3 2024-09-13T14:28:12.875 INFO:tasks.rados.rados.0.smithi026.stdout:2705: writing smithi02683976-39 from 8118272 to 8314880 tid 4 2024-09-13T14:28:12.875 INFO:tasks.rados.rados.0.smithi026.stdout:2700: done (4 left) 2024-09-13T14:28:12.875 INFO:tasks.rados.rados.0.smithi026.stdout:2703: done (3 left) 2024-09-13T14:28:12.876 INFO:tasks.rados.rados.0.smithi026.stdout:2706: read oid 42 snap -1 2024-09-13T14:28:12.876 INFO:tasks.rados.rados.0.smithi026.stdout:2706: expect (ObjNum 826 snap 259 seq_num 826) 2024-09-13T14:28:12.876 INFO:tasks.rados.rados.0.smithi026.stdout:2707: snap_remove snap 252 2024-09-13T14:28:13.235 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 3 to smithi02683976-36 2024-09-13T14:28:13.235 INFO:tasks.rados.rados.0.smithi026.stdout:2701: expect (ObjNum 716 snap 209 seq_num 716) 2024-09-13T14:28:13.705 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 4 to smithi02683976-36 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 5 to smithi02683976-36 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 6 to smithi02683976-36 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2704: finishing write tid 7 to smithi02683976-36 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1478 (ObjNum 846 snap 263 seq_num 846) dirty exists 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2704: left oid 36 (ObjNum 846 snap 263 seq_num 846) 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 1 to smithi02683976-39 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 2 to smithi02683976-39 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 3 to smithi02683976-39 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2701: done (4 left) 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2704: done (3 left) 2024-09-13T14:28:13.706 INFO:tasks.rados.rados.0.smithi026.stdout:2707: done (2 left) 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2708: delete oid 13 current snap is 263 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 4 to smithi02683976-39 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 5 to smithi02683976-39 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2705: finishing write tid 6 to smithi02683976-39 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1257 (ObjNum 847 snap 263 seq_num 847) dirty exists 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2705: left oid 39 (ObjNum 847 snap 263 seq_num 847) 2024-09-13T14:28:13.707 INFO:tasks.rados.rados.0.smithi026.stdout:2706: expect (ObjNum 826 snap 259 seq_num 826) 2024-09-13T14:28:13.991 INFO:tasks.rados.rados.0.smithi026.stdout:2705: done (2 left) 2024-09-13T14:28:13.992 INFO:tasks.rados.rados.0.smithi026.stdout:2706: done (1 left) 2024-09-13T14:28:13.992 INFO:tasks.rados.rados.0.smithi026.stdout:2708: done (0 left) 2024-09-13T14:28:13.992 INFO:tasks.rados.rados.0.smithi026.stdout:2709: delete oid 39 current snap is 263 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2709: done (0 left) 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2710: copy_from oid 18 from oid 48 current snap is 263 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2711: read oid 29 snap -1 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2711: expect (ObjNum 844 snap 263 seq_num 844) 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2712: copy_from oid 17 from oid 1 current snap is 263 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2713: read oid 16 snap -1 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2713: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:28:13.994 INFO:tasks.rados.rados.0.smithi026.stdout:2714: delete oid 11 current snap is 263 2024-09-13T14:28:13.996 INFO:tasks.rados.rados.0.smithi026.stdout:2712: finishing copy_from to smithi02683976-17 2024-09-13T14:28:13.996 INFO:tasks.rados.rados.0.smithi026.stdout:2712: got expected ENOENT (src dne) 2024-09-13T14:28:13.996 INFO:tasks.rados.rados.0.smithi026.stdout:2712: finishing copy_from racing read to smithi02683976-17 2024-09-13T14:28:13.998 INFO:tasks.rados.rados.0.smithi026.stdout:2712: done (4 left) 2024-09-13T14:28:13.998 INFO:tasks.rados.rados.0.smithi026.stdout:2714: done (3 left) 2024-09-13T14:28:13.998 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 263 2024-09-13T14:28:13.998 INFO:tasks.rados.rados.0.smithi026.stdout:2715: seq_num 851 ranges {1605632=655360,2260992=688128,2949120=679936,3629056=237568} 2024-09-13T14:28:14.011 INFO:tasks.rados.rados.0.smithi026.stdout:2715: writing smithi02683976-14 from 1605632 to 2260992 tid 1 2024-09-13T14:28:14.016 INFO:tasks.rados.rados.0.smithi026.stdout:2715: writing smithi02683976-14 from 2260992 to 2949120 tid 2 2024-09-13T14:28:14.020 INFO:tasks.rados.rados.0.smithi026.stdout:2715: writing smithi02683976-14 from 2949120 to 3629056 tid 3 2024-09-13T14:28:14.022 INFO:tasks.rados.rados.0.smithi026.stdout:2715: writing smithi02683976-14 from 3629056 to 3866624 tid 4 2024-09-13T14:28:14.022 INFO:tasks.rados.rados.0.smithi026.stdout:2716: delete oid 46 current snap is 263 2024-09-13T14:28:14.022 INFO:tasks.rados.rados.0.smithi026.stdout:2711: expect (ObjNum 844 snap 263 seq_num 844) 2024-09-13T14:28:14.121 INFO:tasks.rados.rados.0.smithi026.stdout:2713: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:28:14.157 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 1 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 2 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2710: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 3 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 4 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 5 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: finishing write tid 6 to smithi02683976-14 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1279 (ObjNum 851 snap 263 seq_num 851) dirty exists 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2715: left oid 14 (ObjNum 851 snap 263 seq_num 851) 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2710: finishing copy_from to smithi02683976-18 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1132 (ObjNum 817 snap 256 seq_num 817) dirty exists 2024-09-13T14:28:14.158 INFO:tasks.rados.rados.0.smithi026.stdout:2710: done (4 left) 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2711: done (3 left) 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2713: done (2 left) 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2715: done (1 left) 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2716: done (0 left) 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2717: copy_from oid 15 from oid 29 current snap is 263 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 263 2024-09-13T14:28:14.159 INFO:tasks.rados.rados.0.smithi026.stdout:2718: seq_num 853 ranges {0=696320,696320=507904,1204224=647168,1851392=450560} 2024-09-13T14:28:14.161 INFO:tasks.rados.rados.0.smithi026.stdout:2718: writing smithi02683976-4 from 0 to 696320 tid 1 2024-09-13T14:28:14.164 INFO:tasks.rados.rados.0.smithi026.stdout:2718: writing smithi02683976-4 from 696320 to 1204224 tid 2 2024-09-13T14:28:14.168 INFO:tasks.rados.rados.0.smithi026.stdout:2718: writing smithi02683976-4 from 1204224 to 1851392 tid 3 2024-09-13T14:28:14.171 INFO:tasks.rados.rados.0.smithi026.stdout:2718: writing smithi02683976-4 from 1851392 to 2301952 tid 4 2024-09-13T14:28:14.171 INFO:tasks.rados.rados.0.smithi026.stdout:2719: delete oid 41 current snap is 263 2024-09-13T14:28:14.171 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 1 to smithi02683976-4 2024-09-13T14:28:14.172 INFO:tasks.rados.rados.0.smithi026.stdout:2717: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:28:14.172 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 2 to smithi02683976-4 2024-09-13T14:28:14.173 INFO:tasks.rados.rados.0.smithi026.stdout:2719: done (2 left) 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:2720: read oid 17 snap 242 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:2720: expect deleted 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:2721: rmattr oid 16 current snap is 263 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:2721: done (3 left) 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 263 2024-09-13T14:28:14.174 INFO:tasks.rados.rados.0.smithi026.stdout:2722: seq_num 855 ranges {0=729088,729088=671744,1400832=663552,2064384=270336} 2024-09-13T14:28:14.178 INFO:tasks.rados.rados.0.smithi026.stdout:2722: writing smithi02683976-30 from 0 to 729088 tid 1 2024-09-13T14:28:14.182 INFO:tasks.rados.rados.0.smithi026.stdout:2722: writing smithi02683976-30 from 729088 to 1400832 tid 2 2024-09-13T14:28:14.185 INFO:tasks.rados.rados.0.smithi026.stdout:2722: writing smithi02683976-30 from 1400832 to 2064384 tid 3 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:2722: writing smithi02683976-30 from 2064384 to 2334720 tid 4 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:2723: rollback oid 36 current snap is 263 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 235 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:2717: finishing copy_from to smithi02683976-15 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1280 (ObjNum 844 snap 263 seq_num 844) dirty exists 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 3 to smithi02683976-4 2024-09-13T14:28:14.187 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 4 to smithi02683976-4 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 5 to smithi02683976-4 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2718: finishing write tid 6 to smithi02683976-4 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1283 (ObjNum 853 snap 263 seq_num 853) dirty exists 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2718: left oid 4 (ObjNum 853 snap 263 seq_num 853) 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 1 to smithi02683976-30 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 2 to smithi02683976-30 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2717: done (4 left) 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2718: done (3 left) 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2720: done (2 left) 2024-09-13T14:28:14.188 INFO:tasks.rados.rados.0.smithi026.stdout:2724: setattr oid 37 current snap is 263 2024-09-13T14:28:14.189 INFO:tasks.rados.rados.0.smithi026.stdout:2725: rollback oid 12 current snap is 263 2024-09-13T14:28:14.189 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 255 2024-09-13T14:28:14.189 INFO:tasks.rados.rados.0.smithi026.stdout:2726: snap_create 2024-09-13T14:28:14.189 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 3 to smithi02683976-30 2024-09-13T14:28:14.190 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 4 to smithi02683976-30 2024-09-13T14:28:14.190 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 5 to smithi02683976-30 2024-09-13T14:28:14.191 INFO:tasks.rados.rados.0.smithi026.stdout:2723: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:28:14.191 INFO:tasks.rados.rados.0.smithi026.stdout:2725: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:28:14.191 INFO:tasks.rados.rados.0.smithi026.stdout:2723: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:28:14.191 INFO:tasks.rados.rados.0.smithi026.stdout:2725: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1186 (ObjNum 65 snap 0 seq_num 1667064916) dirty dne 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1045 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:2722: finishing write tid 6 to smithi02683976-30 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1288 (ObjNum 855 snap 263 seq_num 855) dirty exists 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:2722: left oid 30 (ObjNum 855 snap 263 seq_num 855) 2024-09-13T14:28:14.192 INFO:tasks.rados.rados.0.smithi026.stdout:2723: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:28:14.193 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1493 (ObjNum 733 snap 215 seq_num 733) dirty exists 2024-09-13T14:28:14.323 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:28:14.531 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: pgmap v489: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-09-13T14:28:14.531 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:28:14.531 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.532 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:14 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:28:14.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: pgmap v489: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-09-13T14:28:14.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: pgmap v489: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:14.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:14 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:28:14.905 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:28:14.905 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 5s ago 11m 18.4M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:28:14.905 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (6m) 22s ago 11m 88.6M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:28:14.905 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (8s) 5s ago 13m 14.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 e163e8cfc713 2024-09-13T14:28:14.905 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (13m) 22s ago 13m 23.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 870132b0654a 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (5m) 22s ago 16m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) 5s ago 18m 529M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 5s ago 18m 66.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (5m) 22s ago 16m 52.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 5s ago 16m 55.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 5s ago 11m 11.1M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (6m) 22s ago 11m 10.6M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 5s ago 16m 242M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (3m) 5s ago 15m 240M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 5s ago 15m 193M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (2m) 5s ago 15m 96.1M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (2m) 22s ago 14m 136M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (96s) 22s ago 14m 118M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 b9422108344c 2024-09-13T14:28:14.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (61s) 22s ago 13m 79.3M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 304ffd9a14d8 2024-09-13T14:28:14.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (26s) 22s ago 13m 11.7M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 606a2c6e152f 2024-09-13T14:28:14.907 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (6m) 22s ago 12m 54.0M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:28:14.971 INFO:tasks.rados.rados.0.smithi026.stdout:2722: done (4 left) 2024-09-13T14:28:14.971 INFO:tasks.rados.rados.0.smithi026.stdout:2723: done (3 left) 2024-09-13T14:28:14.971 INFO:tasks.rados.rados.0.smithi026.stdout:2724: done (2 left) 2024-09-13T14:28:14.972 INFO:tasks.rados.rados.0.smithi026.stdout:2725: done (1 left) 2024-09-13T14:28:14.972 INFO:tasks.rados.rados.0.smithi026.stdout:2726: done (0 left) 2024-09-13T14:28:14.972 INFO:tasks.rados.rados.0.smithi026.stdout:2727: rollback oid 14 current snap is 264 2024-09-13T14:28:14.972 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 239 2024-09-13T14:28:14.972 INFO:tasks.rados.rados.0.smithi026.stdout:2728: snap_create 2024-09-13T14:28:14.975 INFO:tasks.rados.rados.0.smithi026.stdout:2727: finishing rollback tid 0 to smithi02683976-14 2024-09-13T14:28:14.975 INFO:tasks.rados.rados.0.smithi026.stdout:2727: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:28:14.975 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1281 (ObjNum 65 snap 0 seq_num 520205248) dirty dne 2024-09-13T14:28:15.263 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:28:15.263 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:28:15.263 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:28:15.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 8 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 1, 2024-09-13T14:28:15.264 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 14 2024-09-13T14:28:15.265 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:28:15.265 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:28:15.427 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:15 smithi145 ceph-mon[89145]: Detected new or changed devices on smithi026 2024-09-13T14:28:15.427 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:15 smithi145 ceph-mon[89145]: Upgrade: It appears safe to stop mds.b 2024-09-13T14:28:15.428 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:15 smithi145 ceph-mon[89145]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.428 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:15 smithi145 ceph-mon[89145]: osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[99461]: Detected new or changed devices on smithi026 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[99461]: Upgrade: It appears safe to stop mds.b 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[99461]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[99461]: osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[97022]: Detected new or changed devices on smithi026 2024-09-13T14:28:15.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[97022]: Upgrade: It appears safe to stop mds.b 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[97022]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:15 smithi026 ceph-mon[97022]: osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:28:15.572 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "14/20 daemons upgraded", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:28:15.573 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:2727: done (1 left) 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:2728: done (0 left) 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:2729: read oid 49 snap -1 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:2729: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 265 2024-09-13T14:28:15.972 INFO:tasks.rados.rados.0.smithi026.stdout:2730: seq_num 856 ranges {10461184=745472,11206656=557056,11763712=417792,12181504=434176,12615680=196608} 2024-09-13T14:28:16.034 INFO:tasks.rados.rados.0.smithi026.stdout:2730: writing smithi02683976-48 from 10461184 to 11206656 tid 1 2024-09-13T14:28:16.037 INFO:tasks.rados.rados.0.smithi026.stdout:2730: writing smithi02683976-48 from 11206656 to 11763712 tid 2 2024-09-13T14:28:16.039 INFO:tasks.rados.rados.0.smithi026.stdout:2730: writing smithi02683976-48 from 11763712 to 12181504 tid 3 2024-09-13T14:28:16.042 INFO:tasks.rados.rados.0.smithi026.stdout:2730: writing smithi02683976-48 from 12181504 to 12615680 tid 4 2024-09-13T14:28:16.043 INFO:tasks.rados.rados.0.smithi026.stdout:2730: writing smithi02683976-48 from 12615680 to 12812288 tid 5 2024-09-13T14:28:16.043 INFO:tasks.rados.rados.0.smithi026.stdout:2731: rmattr oid 11 current snap is 265 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:2731: done (2 left) 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:2732: read oid 15 snap -1 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:2732: expect (ObjNum 844 snap 263 seq_num 844) 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:2733: rollback oid 34 current snap is 265 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 249 2024-09-13T14:28:16.044 INFO:tasks.rados.rados.0.smithi026.stdout:2729: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:16.323 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 1 to smithi02683976-48 2024-09-13T14:28:16.323 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 2 to smithi02683976-48 2024-09-13T14:28:16.323 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 3 to smithi02683976-48 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 4 to smithi02683976-48 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 5 to smithi02683976-48 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 6 to smithi02683976-48 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: finishing write tid 7 to smithi02683976-48 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1143 (ObjNum 856 snap 265 seq_num 856) dirty exists 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: left oid 48 (ObjNum 856 snap 265 seq_num 856) 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2733: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2729: done (3 left) 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2730: done (2 left) 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2734: snap_create 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2733: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:2733: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:28:16.324 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 860 (ObjNum 793 snap 245 seq_num 793) dirty exists 2024-09-13T14:28:16.325 INFO:tasks.rados.rados.0.smithi026.stdout:2732: expect (ObjNum 844 snap 263 seq_num 844) 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: pgmap v492: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3116796435' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:28:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:16.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:16 smithi145 ceph-mon[89145]: osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:28:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: pgmap v492: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-09-13T14:28:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3116796435' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[97022]: osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: pgmap v492: 121 pgs: 121 active+clean; 316 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3116796435' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:28:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:16.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:16 smithi026 ceph-mon[99461]: osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:28:17.249 INFO:tasks.rados.rados.0.smithi026.stdout:2732: done (2 left) 2024-09-13T14:28:17.249 INFO:tasks.rados.rados.0.smithi026.stdout:2733: done (1 left) 2024-09-13T14:28:17.249 INFO:tasks.rados.rados.0.smithi026.stdout:2734: done (0 left) 2024-09-13T14:28:17.250 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 266 2024-09-13T14:28:17.250 INFO:tasks.rados.rados.0.smithi026.stdout:2735: seq_num 857 ranges {8994816=761856,9756672=442368,10199040=630784,10829824=532480} 2024-09-13T14:28:17.302 INFO:tasks.rados.rados.0.smithi026.stdout:2735: writing smithi02683976-32 from 8994816 to 9756672 tid 1 2024-09-13T14:28:17.305 INFO:tasks.rados.rados.0.smithi026.stdout:2735: writing smithi02683976-32 from 9756672 to 10199040 tid 2 2024-09-13T14:28:17.309 INFO:tasks.rados.rados.0.smithi026.stdout:2735: writing smithi02683976-32 from 10199040 to 10829824 tid 3 2024-09-13T14:28:17.312 INFO:tasks.rados.rados.0.smithi026.stdout:2735: writing smithi02683976-32 from 10829824 to 11362304 tid 4 2024-09-13T14:28:17.313 INFO:tasks.rados.rados.0.smithi026.stdout:2736: snap_remove snap 230 2024-09-13T14:28:17.342 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:17 smithi145 systemd[1]: Stopping Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:17.342 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:17 smithi145 ceph-mon[89145]: Upgrade: Updating mds.b 2024-09-13T14:28:17.342 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:17 smithi145 ceph-mon[89145]: Deploying daemon mds.b on smithi145 2024-09-13T14:28:17.342 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:17 smithi145 ceph-mon[89145]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:17.350 INFO:tasks.rados.rados.0.smithi026.stdout:2736: done (1 left) 2024-09-13T14:28:17.351 INFO:tasks.rados.rados.0.smithi026.stdout:2737: rollback oid 6 current snap is 266 2024-09-13T14:28:17.351 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 249 2024-09-13T14:28:17.352 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 1 to smithi02683976-32 2024-09-13T14:28:17.352 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 2 to smithi02683976-32 2024-09-13T14:28:17.352 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 3 to smithi02683976-32 2024-09-13T14:28:17.352 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 4 to smithi02683976-32 2024-09-13T14:28:17.353 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 5 to smithi02683976-32 2024-09-13T14:28:17.353 INFO:tasks.rados.rados.0.smithi026.stdout:2735: finishing write tid 6 to smithi02683976-32 2024-09-13T14:28:17.353 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1286 (ObjNum 857 snap 266 seq_num 857) dirty exists 2024-09-13T14:28:17.353 INFO:tasks.rados.rados.0.smithi026.stdout:2735: left oid 32 (ObjNum 857 snap 266 seq_num 857) 2024-09-13T14:28:17.353 INFO:tasks.rados.rados.0.smithi026.stdout:2735: done (1 left) 2024-09-13T14:28:17.354 INFO:tasks.rados.rados.0.smithi026.stdout:2738: delete oid 5 current snap is 266 2024-09-13T14:28:17.363 INFO:tasks.rados.rados.0.smithi026.stdout:2738: done (1 left) 2024-09-13T14:28:17.363 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 266 2024-09-13T14:28:17.363 INFO:tasks.rados.rados.0.smithi026.stdout:2739: seq_num 859 ranges {11337728=794624,12132352=614400,12746752=638976,13385728=352256} 2024-09-13T14:28:17.429 INFO:tasks.rados.rados.0.smithi026.stdout:2739: writing smithi02683976-26 from 11337728 to 12132352 tid 1 2024-09-13T14:28:17.432 INFO:tasks.rados.rados.0.smithi026.stdout:2739: writing smithi02683976-26 from 12132352 to 12746752 tid 2 2024-09-13T14:28:17.436 INFO:tasks.rados.rados.0.smithi026.stdout:2739: writing smithi02683976-26 from 12746752 to 13385728 tid 3 2024-09-13T14:28:17.438 INFO:tasks.rados.rados.0.smithi026.stdout:2739: writing smithi02683976-26 from 13385728 to 13737984 tid 4 2024-09-13T14:28:17.438 INFO:tasks.rados.rados.0.smithi026.stdout:2740: snap_remove snap 245 2024-09-13T14:28:17.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[99461]: Upgrade: Updating mds.b 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[99461]: Deploying daemon mds.b on smithi145 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[99461]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[97022]: Upgrade: Updating mds.b 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[97022]: Deploying daemon mds.b on smithi145 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-mon[97022]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:17.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:17 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[96999]: 2024-09-13T14:28:17.341+0000 7fe1fa64b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:17.705 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[58912]: 2024-09-13T14:28:17.339+0000 7fcc7503b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:28:17.705 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[58912]: 2024-09-13T14:28:17.339+0000 7fcc7503b640 -1 mds.b *** got signal Terminated *** 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2737: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2737: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2737: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1358 (ObjNum 794 snap 245 seq_num 794) dirty exists 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 1 to smithi02683976-26 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 2 to smithi02683976-26 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 3 to smithi02683976-26 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 4 to smithi02683976-26 2024-09-13T14:28:18.358 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 5 to smithi02683976-26 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2739: finishing write tid 6 to smithi02683976-26 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1191 (ObjNum 859 snap 266 seq_num 859) dirty exists 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2739: left oid 26 (ObjNum 859 snap 266 seq_num 859) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2737: done (2 left) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2739: done (1 left) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2740: done (0 left) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2741: read oid 32 snap -1 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2741: expect (ObjNum 857 snap 266 seq_num 857) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2742: copy_from oid 46 from oid 28 current snap is 266 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2743: read oid 42 snap 249 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2743: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2744: copy_from oid 39 from oid 49 current snap is 266 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 266 2024-09-13T14:28:18.359 INFO:tasks.rados.rados.0.smithi026.stdout:2745: seq_num 860 ranges {9191424=8192} 2024-09-13T14:28:18.412 INFO:tasks.rados.rados.0.smithi026.stdout:2745: writing smithi02683976-8 from 9191424 to 9199616 tid 1 2024-09-13T14:28:18.412 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 266 2024-09-13T14:28:18.414 INFO:tasks.rados.rados.0.smithi026.stdout:2742: finishing copy_from to smithi02683976-46 2024-09-13T14:28:18.414 INFO:tasks.rados.rados.0.smithi026.stdout:2742: got expected ENOENT (src dne) 2024-09-13T14:28:18.414 INFO:tasks.rados.rados.0.smithi026.stdout:2742: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:28:18.414 INFO:tasks.rados.rados.0.smithi026.stdout:2743: expect (ObjNum 655 snap 184 seq_num 655) 2024-09-13T14:28:18.496 INFO:tasks.rados.rados.0.smithi026.stdout:2741: expect (ObjNum 857 snap 266 seq_num 857) 2024-09-13T14:28:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: pgmap v495: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.0 MiB/s wr, 24 op/s 2024-09-13T14:28:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:28:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[97022]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: pgmap v495: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.0 MiB/s wr, 24 op/s 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:28:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:18.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:28:18.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-13T14:28:18.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:18.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:28:18.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:18 smithi026 ceph-mon[99461]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: pgmap v495: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.0 MiB/s wr, 24 op/s 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:28:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-13T14:28:18.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:28:18.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:28:18.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:18 smithi145 ceph-mon[89145]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-13T14:28:19.303 INFO:tasks.rados.rados.0.smithi026.stdout:2745: finishing write tid 1 to smithi02683976-8 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2745: finishing write tid 2 to smithi02683976-8 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2744: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2744: finishing copy_from to smithi02683976-39 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1174 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2745: finishing write tid 3 to smithi02683976-8 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1282 (ObjNum 860 snap 266 seq_num 860) dirty exists 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2745: left oid 8 (ObjNum 860 snap 266 seq_num 860) 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2746: seq_num 861 ranges {0=24576} 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2746: writing smithi02683976-14 from 0 to 24576 tid 1 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2741: done (5 left) 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2742: done (4 left) 2024-09-13T14:28:19.304 INFO:tasks.rados.rados.0.smithi026.stdout:2743: done (3 left) 2024-09-13T14:28:19.305 INFO:tasks.rados.rados.0.smithi026.stdout:2744: done (2 left) 2024-09-13T14:28:19.305 INFO:tasks.rados.rados.0.smithi026.stdout:2745: done (1 left) 2024-09-13T14:28:19.305 INFO:tasks.rados.rados.0.smithi026.stdout:2747: setattr oid 28 current snap is 266 2024-09-13T14:28:19.305 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 266 2024-09-13T14:28:19.305 INFO:tasks.rados.rados.0.smithi026.stdout:2748: seq_num 862 ranges {9535488=40960} 2024-09-13T14:28:19.357 INFO:tasks.rados.rados.0.smithi026.stdout:2748: writing smithi02683976-3 from 9535488 to 9576448 tid 1 2024-09-13T14:28:19.357 INFO:tasks.rados.rados.0.smithi026.stdout:2749: read oid 16 snap -1 2024-09-13T14:28:19.357 INFO:tasks.rados.rados.0.smithi026.stdout:2749: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:28:19.357 INFO:tasks.rados.rados.0.smithi026.stdout:2750: delete oid 6 current snap is 266 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:2746: finishing write tid 1 to smithi02683976-14 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:2746: finishing write tid 2 to smithi02683976-14 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1180 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:2746: finishing write tid 3 to smithi02683976-14 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1288 (ObjNum 861 snap 266 seq_num 861) dirty exists 2024-09-13T14:28:19.358 INFO:tasks.rados.rados.0.smithi026.stdout:2746: left oid 14 (ObjNum 861 snap 266 seq_num 861) 2024-09-13T14:28:19.360 INFO:tasks.rados.rados.0.smithi026.stdout:2746: done (4 left) 2024-09-13T14:28:19.360 INFO:tasks.rados.rados.0.smithi026.stdout:2747: done (3 left) 2024-09-13T14:28:19.360 INFO:tasks.rados.rados.0.smithi026.stdout:2750: done (2 left) 2024-09-13T14:28:19.360 INFO:tasks.rados.rados.0.smithi026.stdout:2751: read oid 39 snap -1 2024-09-13T14:28:19.361 INFO:tasks.rados.rados.0.smithi026.stdout:2751: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:19.361 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 266 2024-09-13T14:28:19.361 INFO:tasks.rados.rados.0.smithi026.stdout:2752: seq_num 864 ranges {2310144=73728} 2024-09-13T14:28:19.374 INFO:tasks.rados.rados.0.smithi026.stdout:2752: writing smithi02683976-36 from 2310144 to 2383872 tid 1 2024-09-13T14:28:19.374 INFO:tasks.rados.rados.0.smithi026.stdout:2753: delete oid 1 current snap is 266 2024-09-13T14:28:19.374 INFO:tasks.rados.rados.0.smithi026.stdout:2748: finishing write tid 1 to smithi02683976-3 2024-09-13T14:28:19.374 INFO:tasks.rados.rados.0.smithi026.stdout:2749: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2748: finishing write tid 2 to smithi02683976-3 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2748: finishing write tid 3 to smithi02683976-3 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1520 (ObjNum 862 snap 266 seq_num 862) dirty exists 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2748: left oid 3 (ObjNum 862 snap 266 seq_num 862) 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2748: done (4 left) 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2749: done (3 left) 2024-09-13T14:28:19.413 INFO:tasks.rados.rados.0.smithi026.stdout:2753: done (2 left) 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 266 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:2752: finishing write tid 1 to smithi02683976-36 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:2752: finishing write tid 2 to smithi02683976-36 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:2752: finishing write tid 3 to smithi02683976-36 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1523 (ObjNum 864 snap 266 seq_num 864) dirty exists 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:2752: left oid 36 (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:28:19.414 INFO:tasks.rados.rados.0.smithi026.stdout:2751: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:19.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[99461]: osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:28:19.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:reconnect 2024-09-13T14:28:19.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[99461]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-13T14:28:19.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[97022]: osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:28:19.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:reconnect 2024-09-13T14:28:19.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:19 smithi026 ceph-mon[97022]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-13T14:28:19.689 INFO:tasks.rados.rados.0.smithi026.stdout:2754: seq_num 866 ranges {11010048=106496} 2024-09-13T14:28:19.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:19 smithi145 ceph-mon[89145]: osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:28:19.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:19 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:reconnect 2024-09-13T14:28:19.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:19 smithi145 ceph-mon[89145]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-13T14:28:19.750 INFO:tasks.rados.rados.0.smithi026.stdout:2754: writing smithi02683976-23 from 11010048 to 11116544 tid 1 2024-09-13T14:28:19.750 INFO:tasks.rados.rados.0.smithi026.stdout:2751: done (2 left) 2024-09-13T14:28:19.750 INFO:tasks.rados.rados.0.smithi026.stdout:2752: done (1 left) 2024-09-13T14:28:19.750 INFO:tasks.rados.rados.0.smithi026.stdout:2755: copy_from oid 32 from oid 49 current snap is 266 2024-09-13T14:28:19.750 INFO:tasks.rados.rados.0.smithi026.stdout:2756: snap_create 2024-09-13T14:28:19.754 INFO:tasks.rados.rados.0.smithi026.stdout:2754: finishing write tid 1 to smithi02683976-23 2024-09-13T14:28:19.754 INFO:tasks.rados.rados.0.smithi026.stdout:2754: finishing write tid 2 to smithi02683976-23 2024-09-13T14:28:19.755 INFO:tasks.rados.rados.0.smithi026.stdout:2754: finishing write tid 3 to smithi02683976-23 2024-09-13T14:28:19.756 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1290 (ObjNum 866 snap 266 seq_num 866) dirty exists 2024-09-13T14:28:19.756 INFO:tasks.rados.rados.0.smithi026.stdout:2754: left oid 23 (ObjNum 866 snap 266 seq_num 866) 2024-09-13T14:28:19.772 INFO:tasks.rados.rados.0.smithi026.stdout:2755: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:28:19.775 INFO:tasks.rados.rados.0.smithi026.stdout:2755: finishing copy_from to smithi02683976-32 2024-09-13T14:28:19.775 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1174 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:28:20.380 INFO:tasks.rados.rados.0.smithi026.stdout:2754: done (2 left) 2024-09-13T14:28:20.381 INFO:tasks.rados.rados.0.smithi026.stdout:2755: done (1 left) 2024-09-13T14:28:20.381 INFO:tasks.rados.rados.0.smithi026.stdout:2756: done (0 left) 2024-09-13T14:28:20.381 INFO:tasks.rados.rados.0.smithi026.stdout:2757: copy_from oid 4 from oid 16 current snap is 267 2024-09-13T14:28:20.381 INFO:tasks.rados.rados.0.smithi026.stdout:2758: snap_create 2024-09-13T14:28:20.393 INFO:tasks.rados.rados.0.smithi026.stdout:2757: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:28:20.394 INFO:tasks.rados.rados.0.smithi026.stdout:2757: finishing copy_from to smithi02683976-4 2024-09-13T14:28:20.394 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1157 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:28:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[97022]: pgmap v499: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 9.0 MiB/s wr, 27 op/s 2024-09-13T14:28:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:rejoin 2024-09-13T14:28:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[97022]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-13T14:28:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[97022]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-13T14:28:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[99461]: pgmap v499: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 9.0 MiB/s wr, 27 op/s 2024-09-13T14:28:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:rejoin 2024-09-13T14:28:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[99461]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-13T14:28:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:20 smithi026 ceph-mon[99461]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-13T14:28:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:20 smithi145 ceph-mon[89145]: pgmap v499: 121 pgs: 121 active+clean; 340 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 9.0 MiB/s wr, 27 op/s 2024-09-13T14:28:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:20 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:rejoin 2024-09-13T14:28:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:20 smithi145 ceph-mon[89145]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-13T14:28:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:20 smithi145 ceph-mon[89145]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-13T14:28:20.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:21.386 INFO:tasks.rados.rados.0.smithi026.stdout:2757: done (1 left) 2024-09-13T14:28:21.386 INFO:tasks.rados.rados.0.smithi026.stdout:2758: done (0 left) 2024-09-13T14:28:21.386 INFO:tasks.rados.rados.0.smithi026.stdout:2759: read oid 49 snap -1 2024-09-13T14:28:21.386 INFO:tasks.rados.rados.0.smithi026.stdout:2759: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:21.386 INFO:tasks.rados.rados.0.smithi026.stdout:2760: delete oid 45 current snap is 268 2024-09-13T14:28:21.391 INFO:tasks.rados.rados.0.smithi026.stdout:2760: done (1 left) 2024-09-13T14:28:21.391 INFO:tasks.rados.rados.0.smithi026.stdout:2761: snap_remove snap 265 2024-09-13T14:28:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:21] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.51.0" 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[97022]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:active 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=a=up:active} 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[97022]: osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[99461]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:28:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:28:21.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:active 2024-09-13T14:28:21.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=a=up:active} 2024-09-13T14:28:21.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:21 smithi026 ceph-mon[99461]: osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:28:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:21 smithi145 ceph-mon[89145]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:28:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:21 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:28:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:21 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.26:6834/3923891201,v1:172.21.15.26:6835/3923891201] up:active 2024-09-13T14:28:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:21 smithi145 ceph-mon[89145]: fsmap cephfs:1 {0=a=up:active} 2024-09-13T14:28:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:21 smithi145 ceph-mon[89145]: osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:28:22.391 INFO:tasks.rados.rados.0.smithi026.stdout:2761: done (1 left) 2024-09-13T14:28:22.391 INFO:tasks.rados.rados.0.smithi026.stdout:2762: rmattr oid 46 current snap is 268 2024-09-13T14:28:22.391 INFO:tasks.rados.rados.0.smithi026.stdout:2762: done (1 left) 2024-09-13T14:28:22.391 INFO:tasks.rados.rados.0.smithi026.stdout:2763: snap_create 2024-09-13T14:28:22.391 INFO:tasks.rados.rados.0.smithi026.stdout:2759: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:22 smithi026 ceph-mon[97022]: pgmap v501: 121 pgs: 121 active+clean; 353 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-09-13T14:28:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:22 smithi026 ceph-mon[97022]: osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:28:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:22 smithi026 ceph-mon[99461]: pgmap v501: 121 pgs: 121 active+clean; 353 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-09-13T14:28:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:22 smithi026 ceph-mon[99461]: osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:28:22.666 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:28:22 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: 2024-09-13T14:28:22.258+0000 7fa9ce433640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-13T14:28:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:22 smithi145 ceph-mon[89145]: pgmap v501: 121 pgs: 121 active+clean; 353 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-09-13T14:28:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:22 smithi145 ceph-mon[89145]: osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:28:23.396 INFO:tasks.rados.rados.0.smithi026.stdout:2759: done (1 left) 2024-09-13T14:28:23.396 INFO:tasks.rados.rados.0.smithi026.stdout:2763: done (0 left) 2024-09-13T14:28:23.396 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 269 2024-09-13T14:28:23.397 INFO:tasks.rados.rados.0.smithi026.stdout:2764: seq_num 868 ranges {1376256=139264} 2024-09-13T14:28:23.404 INFO:tasks.rados.rados.0.smithi026.stdout:2764: writing smithi02683976-4 from 1376256 to 1515520 tid 1 2024-09-13T14:28:23.404 INFO:tasks.rados.rados.0.smithi026.stdout:2765: read oid 38 snap -1 2024-09-13T14:28:23.404 INFO:tasks.rados.rados.0.smithi026.stdout:2765: expect deleted 2024-09-13T14:28:23.405 INFO:tasks.rados.rados.0.smithi026.stdout:2766: rmattr oid 36 current snap is 269 2024-09-13T14:28:23.405 INFO:tasks.rados.rados.0.smithi026.stdout:2766: done (2 left) 2024-09-13T14:28:23.405 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 269 2024-09-13T14:28:23.405 INFO:tasks.rados.rados.0.smithi026.stdout:2767: seq_num 869 ranges {1892352=155648} 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2767: writing smithi02683976-47 from 1892352 to 2048000 tid 1 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2768: delete oid 2 current snap is 269 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2764: finishing write tid 1 to smithi02683976-4 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2764: finishing write tid 2 to smithi02683976-4 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2764: finishing write tid 3 to smithi02683976-4 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1290 (ObjNum 868 snap 269 seq_num 868) dirty exists 2024-09-13T14:28:23.416 INFO:tasks.rados.rados.0.smithi026.stdout:2764: left oid 4 (ObjNum 868 snap 269 seq_num 868) 2024-09-13T14:28:23.419 INFO:tasks.rados.rados.0.smithi026.stdout:2767: finishing write tid 1 to smithi02683976-47 2024-09-13T14:28:23.419 INFO:tasks.rados.rados.0.smithi026.stdout:2764: done (3 left) 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:2765: done (2 left) 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:2768: done (1 left) 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:2769: read oid 19 snap -1 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:2769: expect (ObjNum 802 snap 248 seq_num 802) 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 269 2024-09-13T14:28:23.420 INFO:tasks.rados.rados.0.smithi026.stdout:2770: seq_num 871 ranges {1515520=188416} 2024-09-13T14:28:23.429 INFO:tasks.rados.rados.0.smithi026.stdout:2770: writing smithi02683976-4 from 1515520 to 1703936 tid 1 2024-09-13T14:28:23.429 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 269 2024-09-13T14:28:23.429 INFO:tasks.rados.rados.0.smithi026.stdout:2767: finishing write tid 2 to smithi02683976-47 2024-09-13T14:28:23.429 INFO:tasks.rados.rados.0.smithi026.stdout:2771: seq_num 872 ranges {3268608=212992} 2024-09-13T14:28:23.449 INFO:tasks.rados.rados.0.smithi026.stdout:2771: writing smithi02683976-25 from 3268608 to 3481600 tid 1 2024-09-13T14:28:23.449 INFO:tasks.rados.rados.0.smithi026.stdout:2772: snap_create 2024-09-13T14:28:23.449 INFO:tasks.rados.rados.0.smithi026.stdout:2767: finishing write tid 3 to smithi02683976-47 2024-09-13T14:28:23.449 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1145 (ObjNum 869 snap 269 seq_num 869) dirty exists 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2767: left oid 47 (ObjNum 869 snap 269 seq_num 869) 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2770: finishing write tid 1 to smithi02683976-4 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2770: finishing write tid 2 to smithi02683976-4 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2770: finishing write tid 3 to smithi02683976-4 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1292 (ObjNum 871 snap 269 seq_num 871) dirty exists 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2770: left oid 4 (ObjNum 871 snap 269 seq_num 871) 2024-09-13T14:28:23.450 INFO:tasks.rados.rados.0.smithi026.stdout:2769: expect (ObjNum 802 snap 248 seq_num 802) 2024-09-13T14:28:23.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:23 smithi026 ceph-mon[97022]: osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:28:23.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:23 smithi026 ceph-mon[99461]: osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:28:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:23 smithi145 ceph-mon[89145]: osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:28:24.009 INFO:tasks.rados.rados.0.smithi026.stdout:2771: finishing write tid 1 to smithi02683976-25 2024-09-13T14:28:24.009 INFO:tasks.rados.rados.0.smithi026.stdout:2771: finishing write tid 2 to smithi02683976-25 2024-09-13T14:28:24.009 INFO:tasks.rados.rados.0.smithi026.stdout:2771: finishing write tid 3 to smithi02683976-25 2024-09-13T14:28:24.009 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1261 (ObjNum 872 snap 269 seq_num 872) dirty exists 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2771: left oid 25 (ObjNum 872 snap 269 seq_num 872) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2767: done (4 left) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2769: done (3 left) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2770: done (2 left) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2771: done (1 left) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2772: done (0 left) 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2773: rollback oid 19 current snap is 270 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 235 2024-09-13T14:28:24.010 INFO:tasks.rados.rados.0.smithi026.stdout:2774: snap_create 2024-09-13T14:28:24.016 INFO:tasks.rados.rados.0.smithi026.stdout:2773: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:28:24.019 INFO:tasks.rados.rados.0.smithi026.stdout:2773: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:28:24.020 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1311 (ObjNum 33 snap 0 seq_num 518876432) dirty dne 2024-09-13T14:28:24.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[97022]: pgmap v504: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.4 MiB/s wr, 25 op/s 2024-09-13T14:28:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[97022]: osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:28:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[97022]: osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:28:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[99461]: pgmap v504: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.4 MiB/s wr, 25 op/s 2024-09-13T14:28:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[99461]: osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:28:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:24 smithi026 ceph-mon[99461]: osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:28:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:24 smithi145 ceph-mon[89145]: pgmap v504: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.4 MiB/s wr, 25 op/s 2024-09-13T14:28:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:24 smithi145 ceph-mon[89145]: osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:28:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:24 smithi145 ceph-mon[89145]: osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:28:24.980 INFO:tasks.rados.rados.0.smithi026.stdout:2773: done (1 left) 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:2774: done (0 left) 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:2775: rollback oid 43 current snap is 271 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 43 to 268 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:2776: copy_from oid 32 from oid 46 current snap is 271 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:2777: copy_from oid 33 from oid 40 current snap is 271 2024-09-13T14:28:24.981 INFO:tasks.rados.rados.0.smithi026.stdout:2778: snap_remove snap 242 2024-09-13T14:28:25.985 INFO:tasks.rados.rados.0.smithi026.stdout:2778: done (3 left) 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2779: delete oid 29 current snap is 271 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2776: finishing copy_from to smithi02683976-32 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2776: got expected ENOENT (src dne) 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2775: finishing rollback tid 0 to smithi02683976-43 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2776: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2775: finishing rollback tid 1 to smithi02683976-43 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2775: finishing rollback tid 2 to smithi02683976-43 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1315 (ObjNum 845 snap 263 seq_num 845) dirty exists 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2777: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:2777: finishing copy_from to smithi02683976-33 2024-09-13T14:28:25.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1171 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-13T14:28:25.991 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:25 smithi145 podman[115097]: 2024-09-13 14:28:25.714347066 +0000 UTC m=+8.476287980 container died 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T14:28:25.998 INFO:tasks.rados.rados.0.smithi026.stdout:2775: done (3 left) 2024-09-13T14:28:25.998 INFO:tasks.rados.rados.0.smithi026.stdout:2776: done (2 left) 2024-09-13T14:28:25.998 INFO:tasks.rados.rados.0.smithi026.stdout:2777: done (1 left) 2024-09-13T14:28:25.998 INFO:tasks.rados.rados.0.smithi026.stdout:2779: done (0 left) 2024-09-13T14:28:25.999 INFO:tasks.rados.rados.0.smithi026.stdout:2780: delete oid 21 current snap is 271 2024-09-13T14:28:26.004 INFO:tasks.rados.rados.0.smithi026.stdout:2780: done (0 left) 2024-09-13T14:28:26.004 INFO:tasks.rados.rados.0.smithi026.stdout:2781: copy_from oid 13 from oid 17 current snap is 271 2024-09-13T14:28:26.004 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 271 2024-09-13T14:28:26.004 INFO:tasks.rados.rados.0.smithi026.stdout:2782: seq_num 875 ranges {13737984=262144} 2024-09-13T14:28:26.081 INFO:tasks.rados.rados.0.smithi026.stdout:2782: writing smithi02683976-26 from 13737984 to 14000128 tid 1 2024-09-13T14:28:26.082 INFO:tasks.rados.rados.0.smithi026.stdout:2783: snap_create 2024-09-13T14:28:26.082 INFO:tasks.rados.rados.0.smithi026.stdout:2781: finishing copy_from to smithi02683976-13 2024-09-13T14:28:26.082 INFO:tasks.rados.rados.0.smithi026.stdout:2781: got expected ENOENT (src dne) 2024-09-13T14:28:26.082 INFO:tasks.rados.rados.0.smithi026.stdout:2781: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:28:26.085 INFO:tasks.rados.rados.0.smithi026.stdout:2782: finishing write tid 1 to smithi02683976-26 2024-09-13T14:28:26.085 INFO:tasks.rados.rados.0.smithi026.stdout:2782: finishing write tid 2 to smithi02683976-26 2024-09-13T14:28:26.086 INFO:tasks.rados.rados.0.smithi026.stdout:2782: finishing write tid 3 to smithi02683976-26 2024-09-13T14:28:26.086 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1193 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:28:26.086 INFO:tasks.rados.rados.0.smithi026.stdout:2782: left oid 26 (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:28:26.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:25 smithi026 ceph-mon[97022]: pgmap v507: 121 pgs: 121 active+clean; 357 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-13T14:28:26.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:25 smithi026 ceph-mon[97022]: osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:28:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:25 smithi026 ceph-mon[99461]: pgmap v507: 121 pgs: 121 active+clean; 357 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-13T14:28:26.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:25 smithi026 ceph-mon[99461]: osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:28:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:25 smithi145 ceph-mon[89145]: pgmap v507: 121 pgs: 121 active+clean; 357 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-13T14:28:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:25 smithi145 ceph-mon[89145]: osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:28:26.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 podman[115097]: 2024-09-13 14:28:26.196263557 +0000 UTC m=+8.958204470 container cleanup 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T14:28:26.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 bash[115097]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b 2024-09-13T14:28:26.834 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 podman[115113]: 2024-09-13 14:28:26.456071707 +0000 UTC m=+0.737369229 container remove 870132b0654aac4df4db3a5be789a8b1aec2b4dab821d8d94581b032fedde378 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T14:28:26.834 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.b.service: Deactivated successfully. 2024-09-13T14:28:26.834 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 systemd[1]: Stopped Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:26.834 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.b.service: Consumed 1.748s CPU time. 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2781: done (2 left) 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2782: done (1 left) 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2783: done (0 left) 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2784: copy_from oid 24 from oid 35 current snap is 272 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2785: read oid 32 snap 235 2024-09-13T14:28:27.001 INFO:tasks.rados.rados.0.smithi026.stdout:2785: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:28:27.002 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 272 2024-09-13T14:28:27.002 INFO:tasks.rados.rados.0.smithi026.stdout:2786: seq_num 876 ranges {0=278528} 2024-09-13T14:28:27.002 INFO:tasks.rados.rados.0.smithi026.stdout:2786: writing smithi02683976-41 from 0 to 278528 tid 1 2024-09-13T14:28:27.002 INFO:tasks.rados.rados.0.smithi026.stdout:2787: read oid 40 snap -1 2024-09-13T14:28:27.003 INFO:tasks.rados.rados.0.smithi026.stdout:2787: expect (ObjNum 813 snap 254 seq_num 813) 2024-09-13T14:28:27.003 INFO:tasks.rados.rados.0.smithi026.stdout:2788: snap_create 2024-09-13T14:28:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:2784: finishing copy_from to smithi02683976-24 2024-09-13T14:28:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:2784: got expected ENOENT (src dne) 2024-09-13T14:28:27.005 INFO:tasks.rados.rados.0.smithi026.stdout:2784: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:28:27.007 INFO:tasks.rados.rados.0.smithi026.stdout:2786: finishing write tid 1 to smithi02683976-41 2024-09-13T14:28:27.007 INFO:tasks.rados.rados.0.smithi026.stdout:2786: finishing write tid 2 to smithi02683976-41 2024-09-13T14:28:27.008 INFO:tasks.rados.rados.0.smithi026.stdout:2786: finishing write tid 3 to smithi02683976-41 2024-09-13T14:28:27.008 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 862 (ObjNum 876 snap 272 seq_num 876) dirty exists 2024-09-13T14:28:27.009 INFO:tasks.rados.rados.0.smithi026.stdout:2786: left oid 41 (ObjNum 876 snap 272 seq_num 876) 2024-09-13T14:28:27.010 INFO:tasks.rados.rados.0.smithi026.stdout:2787: expect (ObjNum 813 snap 254 seq_num 813) 2024-09-13T14:28:27.054 INFO:tasks.rados.rados.0.smithi026.stdout:2785: expect (ObjNum 670 snap 188 seq_num 670) 2024-09-13T14:28:27.151 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:26 smithi145 ceph-mon[89145]: osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:28:27.151 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.151 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:26 smithi145 systemd[1]: Starting Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:27.151 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 podman[115248]: 2024-09-13 14:28:27.040612399 +0000 UTC m=+0.111598078 container create 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T14:28:27.151 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 podman[115248]: 2024-09-13 14:28:26.944771892 +0000 UTC m=+0.015757581 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:28:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:26 smithi026 ceph-mon[99461]: osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:28:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:26 smithi026 ceph-mon[97022]: osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:28:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.705 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 podman[115248]: 2024-09-13 14:28:27.293450417 +0000 UTC m=+0.364436105 container init 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T14:28:27.705 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 podman[115248]: 2024-09-13 14:28:27.298911105 +0000 UTC m=+0.369896788 container start 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T14:28:27.705 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[115263]: starting mds.b at 2024-09-13T14:28:27.706 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 bash[115248]: 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 2024-09-13T14:28:27.706 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:28:27 smithi145 systemd[1]: Started Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2784: done (4 left) 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2785: done (3 left) 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2786: done (2 left) 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2787: done (1 left) 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2788: done (0 left) 2024-09-13T14:28:28.006 INFO:tasks.rados.rados.0.smithi026.stdout:2789: snap_remove snap 272 2024-09-13T14:28:28.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:28 smithi145 ceph-mon[89145]: pgmap v510: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 850 KiB/s wr, 21 op/s 2024-09-13T14:28:28.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:28 smithi145 ceph-mon[89145]: osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:28:28.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:28 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:28 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.255 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:28 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[97022]: pgmap v510: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 850 KiB/s wr, 21 op/s 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[97022]: osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[99461]: pgmap v510: 121 pgs: 121 active+clean; 359 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 850 KiB/s wr, 21 op/s 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[99461]: osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:28 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:28.976 INFO:tasks.rados.rados.0.smithi026.stdout:2789: done (0 left) 2024-09-13T14:28:28.976 INFO:tasks.rados.rados.0.smithi026.stdout:2790: read oid 1 snap -1 2024-09-13T14:28:28.977 INFO:tasks.rados.rados.0.smithi026.stdout:2790: expect deleted 2024-09-13T14:28:28.977 INFO:tasks.rados.rados.0.smithi026.stdout:2791: snap_remove snap 262 2024-09-13T14:28:29.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.145:6832/1802307086,v1:172.21.15.145:6833/1802307086] up:boot 2024-09-13T14:28:29.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[97022]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[97022]: osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[97022]: osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.145:6832/1802307086,v1:172.21.15.145:6833/1802307086] up:boot 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[99461]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[99461]: osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:28:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:29 smithi026 ceph-mon[99461]: osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:28:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:29 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.145:6832/1802307086,v1:172.21.15.145:6833/1802307086] up:boot 2024-09-13T14:28:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:29 smithi145 ceph-mon[89145]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-13T14:28:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:29 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-13T14:28:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:29 smithi145 ceph-mon[89145]: osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:28:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:29 smithi145 ceph-mon[89145]: osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:28:29.977 INFO:tasks.rados.rados.0.smithi026.stdout:2791: done (1 left) 2024-09-13T14:28:29.977 INFO:tasks.rados.rados.0.smithi026.stdout:2792: delete oid 42 current snap is 273 2024-09-13T14:28:29.986 INFO:tasks.rados.rados.0.smithi026.stdout:2790: done (1 left) 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:2792: done (0 left) 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:2793: rollback oid 18 current snap is 273 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 255 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:2794: rollback oid 20 current snap is 273 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 266 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 273 2024-09-13T14:28:29.987 INFO:tasks.rados.rados.0.smithi026.stdout:2795: seq_num 878 ranges {0=311296} 2024-09-13T14:28:29.989 INFO:tasks.rados.rados.0.smithi026.stdout:2795: writing smithi02683976-17 from 0 to 311296 tid 1 2024-09-13T14:28:29.989 INFO:tasks.rados.rados.0.smithi026.stdout:2796: read oid 48 snap -1 2024-09-13T14:28:29.989 INFO:tasks.rados.rados.0.smithi026.stdout:2796: expect (ObjNum 856 snap 265 seq_num 856) 2024-09-13T14:28:29.989 INFO:tasks.rados.rados.0.smithi026.stdout:2797: snap_create 2024-09-13T14:28:29.989 INFO:tasks.rados.rados.0.smithi026.stdout:2794: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:28:29.990 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 0 (ObjNum 33 snap 0 seq_num 519096016) dirty dne 2024-09-13T14:28:29.995 INFO:tasks.rados.rados.0.smithi026.stdout:2795: finishing write tid 1 to smithi02683976-17 2024-09-13T14:28:29.998 INFO:tasks.rados.rados.0.smithi026.stdout:2795: finishing write tid 2 to smithi02683976-17 2024-09-13T14:28:30.000 INFO:tasks.rados.rados.0.smithi026.stdout:2795: finishing write tid 3 to smithi02683976-17 2024-09-13T14:28:30.000 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 864 (ObjNum 878 snap 273 seq_num 878) dirty exists 2024-09-13T14:28:30.000 INFO:tasks.rados.rados.0.smithi026.stdout:2795: left oid 17 (ObjNum 878 snap 273 seq_num 878) 2024-09-13T14:28:30.005 INFO:tasks.rados.rados.0.smithi026.stdout:2793: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:28:30.005 INFO:tasks.rados.rados.0.smithi026.stdout:2793: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:28:30.005 INFO:tasks.rados.rados.0.smithi026.stdout:2793: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:28:30.005 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1293 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:28:30.038 INFO:tasks.rados.rados.0.smithi026.stdout:2796: expect (ObjNum 856 snap 265 seq_num 856) 2024-09-13T14:28:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:30 smithi026 ceph-mon[97022]: pgmap v513: 121 pgs: 121 active+clean; 360 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 902 KiB/s wr, 19 op/s 2024-09-13T14:28:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:30 smithi026 ceph-mon[97022]: osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:28:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:30 smithi026 ceph-mon[99461]: pgmap v513: 121 pgs: 121 active+clean; 360 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 902 KiB/s wr, 19 op/s 2024-09-13T14:28:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:30 smithi026 ceph-mon[99461]: osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:28:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:30 smithi145 ceph-mon[89145]: pgmap v513: 121 pgs: 121 active+clean; 360 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 902 KiB/s wr, 19 op/s 2024-09-13T14:28:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:30 smithi145 ceph-mon[89145]: osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:28:30.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:2793: done (4 left) 2024-09-13T14:28:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:2794: done (3 left) 2024-09-13T14:28:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:2795: done (2 left) 2024-09-13T14:28:31.052 INFO:tasks.rados.rados.0.smithi026.stdout:2796: done (1 left) 2024-09-13T14:28:31.052 INFO:tasks.rados.rados.0.smithi026.stdout:2797: done (0 left) 2024-09-13T14:28:31.052 INFO:tasks.rados.rados.0.smithi026.stdout:2798: snap_remove snap 261 2024-09-13T14:28:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:31] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.51.0" 2024-09-13T14:28:32.013 INFO:tasks.rados.rados.0.smithi026.stdout:2798: done (0 left) 2024-09-13T14:28:32.014 INFO:tasks.rados.rados.0.smithi026.stdout:2799: copy_from oid 14 from oid 19 current snap is 274 2024-09-13T14:28:32.014 INFO:tasks.rados.rados.0.smithi026.stdout:2800: rollback oid 13 current snap is 274 2024-09-13T14:28:32.014 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 247 2024-09-13T14:28:32.014 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 274 2024-09-13T14:28:32.014 INFO:tasks.rados.rados.0.smithi026.stdout:2801: seq_num 879 ranges {9576448=327680} 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2801: writing smithi02683976-3 from 9576448 to 9904128 tid 1 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2802: delete oid 43 current snap is 274 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2799: finishing copy_from to smithi02683976-14 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2799: got expected ENOENT (src dne) 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2799: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:28:32.069 INFO:tasks.rados.rados.0.smithi026.stdout:2800: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:28:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:2800: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:28:32.070 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1335 (ObjNum 766 snap 231 seq_num 766) dirty exists 2024-09-13T14:28:32.071 INFO:tasks.rados.rados.0.smithi026.stdout:2799: done (3 left) 2024-09-13T14:28:32.071 INFO:tasks.rados.rados.0.smithi026.stdout:2800: done (2 left) 2024-09-13T14:28:32.071 INFO:tasks.rados.rados.0.smithi026.stdout:2802: done (1 left) 2024-09-13T14:28:32.071 INFO:tasks.rados.rados.0.smithi026.stdout:2803: snap_create 2024-09-13T14:28:32.073 INFO:tasks.rados.rados.0.smithi026.stdout:2801: finishing write tid 1 to smithi02683976-3 2024-09-13T14:28:32.073 INFO:tasks.rados.rados.0.smithi026.stdout:2801: finishing write tid 2 to smithi02683976-3 2024-09-13T14:28:32.074 INFO:tasks.rados.rados.0.smithi026.stdout:2801: finishing write tid 3 to smithi02683976-3 2024-09-13T14:28:32.074 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1567 (ObjNum 879 snap 274 seq_num 879) dirty exists 2024-09-13T14:28:32.074 INFO:tasks.rados.rados.0.smithi026.stdout:2801: left oid 3 (ObjNum 879 snap 274 seq_num 879) 2024-09-13T14:28:32.227 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: pgmap v516: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 260 KiB/s wr, 6 op/s 2024-09-13T14:28:32.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:28:32.258 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.258 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.258 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.258 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:31 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: pgmap v516: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 260 KiB/s wr, 6 op/s 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: pgmap v516: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 260 KiB/s wr, 6 op/s 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:28:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:32.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:31 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:33.018 INFO:tasks.rados.rados.0.smithi026.stdout:2801: done (1 left) 2024-09-13T14:28:33.018 INFO:tasks.rados.rados.0.smithi026.stdout:2803: done (0 left) 2024-09-13T14:28:33.018 INFO:tasks.rados.rados.0.smithi026.stdout:2804: read oid 11 snap -1 2024-09-13T14:28:33.018 INFO:tasks.rados.rados.0.smithi026.stdout:2804: expect deleted 2024-09-13T14:28:33.018 INFO:tasks.rados.rados.0.smithi026.stdout:2805: snap_remove snap 259 2024-09-13T14:28:33.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:33 smithi145 ceph-mon[89145]: osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:28:33.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:33 smithi026 ceph-mon[97022]: osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:28:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:33 smithi026 ceph-mon[99461]: osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:28:33.977 INFO:tasks.rados.rados.0.smithi026.stdout:2804: done (1 left) 2024-09-13T14:28:33.977 INFO:tasks.rados.rados.0.smithi026.stdout:2805: done (0 left) 2024-09-13T14:28:33.977 INFO:tasks.rados.rados.0.smithi026.stdout:2806: setattr oid 40 current snap is 275 2024-09-13T14:28:33.979 INFO:tasks.rados.rados.0.smithi026.stdout:2807: read oid 46 snap -1 2024-09-13T14:28:33.979 INFO:tasks.rados.rados.0.smithi026.stdout:2807: expect deleted 2024-09-13T14:28:33.979 INFO:tasks.rados.rados.0.smithi026.stdout:2808: rollback oid 2 current snap is 275 2024-09-13T14:28:33.979 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 273 2024-09-13T14:28:33.979 INFO:tasks.rados.rados.0.smithi026.stdout:2809: delete oid 42 current snap is 275 2024-09-13T14:28:33.988 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1181 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-13T14:28:33.991 INFO:tasks.rados.rados.0.smithi026.stdout:2806: done (3 left) 2024-09-13T14:28:33.991 INFO:tasks.rados.rados.0.smithi026.stdout:2807: done (2 left) 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:2809: done (1 left) 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:2810: rollback oid 47 current snap is 275 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 264 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:2808: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1259 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:2808: done (1 left) 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 275 2024-09-13T14:28:33.992 INFO:tasks.rados.rados.0.smithi026.stdout:2811: seq_num 882 ranges {0=376832} 2024-09-13T14:28:33.994 INFO:tasks.rados.rados.0.smithi026.stdout:2811: writing smithi02683976-46 from 0 to 376832 tid 1 2024-09-13T14:28:33.994 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 275 2024-09-13T14:28:33.994 INFO:tasks.rados.rados.0.smithi026.stdout:2812: seq_num 883 ranges {24576=393216} 2024-09-13T14:28:33.997 INFO:tasks.rados.rados.0.smithi026.stdout:2812: writing smithi02683976-14 from 24576 to 417792 tid 1 2024-09-13T14:28:33.998 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 275 2024-09-13T14:28:33.998 INFO:tasks.rados.rados.0.smithi026.stdout:2813: seq_num 884 ranges {11337728=409600} 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2813: writing smithi02683976-34 from 11337728 to 11747328 tid 1 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2814: snap_create 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2810: finishing rollback tid 0 to smithi02683976-47 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2810: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2810: finishing rollback tid 2 to smithi02683976-47 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1148 (ObjNum 829 snap 259 seq_num 829) dirty exists 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2811: finishing write tid 1 to smithi02683976-46 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2811: finishing write tid 2 to smithi02683976-46 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2811: finishing write tid 3 to smithi02683976-46 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 944 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:28:34.066 INFO:tasks.rados.rados.0.smithi026.stdout:2811: left oid 46 (ObjNum 882 snap 275 seq_num 882) 2024-09-13T14:28:34.067 INFO:tasks.rados.rados.0.smithi026.stdout:2812: finishing write tid 1 to smithi02683976-14 2024-09-13T14:28:34.067 INFO:tasks.rados.rados.0.smithi026.stdout:2812: finishing write tid 2 to smithi02683976-14 2024-09-13T14:28:34.067 INFO:tasks.rados.rados.0.smithi026.stdout:2812: finishing write tid 3 to smithi02683976-14 2024-09-13T14:28:34.067 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1295 (ObjNum 883 snap 275 seq_num 883) dirty exists 2024-09-13T14:28:34.067 INFO:tasks.rados.rados.0.smithi026.stdout:2812: left oid 14 (ObjNum 883 snap 275 seq_num 883) 2024-09-13T14:28:34.068 INFO:tasks.rados.rados.0.smithi026.stdout:2813: finishing write tid 1 to smithi02683976-34 2024-09-13T14:28:34.069 INFO:tasks.rados.rados.0.smithi026.stdout:2813: finishing write tid 2 to smithi02683976-34 2024-09-13T14:28:34.071 INFO:tasks.rados.rados.0.smithi026.stdout:2813: finishing write tid 3 to smithi02683976-34 2024-09-13T14:28:34.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 866 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:28:34.071 INFO:tasks.rados.rados.0.smithi026.stdout:2813: left oid 34 (ObjNum 884 snap 275 seq_num 884) 2024-09-13T14:28:34.359 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:34 smithi145 ceph-mon[89145]: pgmap v519: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 208 KiB/s wr, 5 op/s 2024-09-13T14:28:34.359 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:34 smithi145 ceph-mon[89145]: osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:28:34.359 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:34 smithi145 ceph-mon[89145]: osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:28:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[99461]: pgmap v519: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 208 KiB/s wr, 5 op/s 2024-09-13T14:28:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[99461]: osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:28:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[99461]: osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:28:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[97022]: pgmap v519: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 208 KiB/s wr, 5 op/s 2024-09-13T14:28:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[97022]: osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:28:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:34 smithi026 ceph-mon[97022]: osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:28:35.027 INFO:tasks.rados.rados.0.smithi026.stdout:2810: done (4 left) 2024-09-13T14:28:35.027 INFO:tasks.rados.rados.0.smithi026.stdout:2811: done (3 left) 2024-09-13T14:28:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:2812: done (2 left) 2024-09-13T14:28:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:2813: done (1 left) 2024-09-13T14:28:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:2814: done (0 left) 2024-09-13T14:28:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:2815: copy_from oid 48 from oid 43 current snap is 276 2024-09-13T14:28:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:2816: setattr oid 7 current snap is 276 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:2817: read oid 36 snap 256 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:2817: expect (ObjNum 738 snap 218 seq_num 738) 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:2818: read oid 25 snap -1 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:2818: expect (ObjNum 872 snap 269 seq_num 872) 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 276 2024-09-13T14:28:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:2819: seq_num 885 ranges {0=425984} 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:2819: writing smithi02683976-44 from 0 to 425984 tid 1 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 276 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:2815: finishing copy_from to smithi02683976-48 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:2815: got expected ENOENT (src dne) 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:2815: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:28:35.032 INFO:tasks.rados.rados.0.smithi026.stdout:2820: seq_num 886 ranges {0=442368} 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2820: writing smithi02683976-24 from 0 to 442368 tid 1 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2815: done (5 left) 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2821: rmattr oid 35 current snap is 276 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2821: done (5 left) 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2822: rmattr oid 49 current snap is 276 2024-09-13T14:28:35.036 INFO:tasks.rados.rados.0.smithi026.stdout:2817: expect (ObjNum 738 snap 218 seq_num 738) 2024-09-13T14:28:35.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1134 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-13T14:28:35.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1182 (ObjNum 703 snap 202 seq_num 703) dirty exists 2024-09-13T14:28:35.051 INFO:tasks.rados.rados.0.smithi026.stdout:2819: finishing write tid 1 to smithi02683976-44 2024-09-13T14:28:35.051 INFO:tasks.rados.rados.0.smithi026.stdout:2819: finishing write tid 2 to smithi02683976-44 2024-09-13T14:28:35.051 INFO:tasks.rados.rados.0.smithi026.stdout:2816: done (5 left) 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2817: done (4 left) 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2822: done (3 left) 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2823: read oid 47 snap 274 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2823: expect (ObjNum 869 snap 269 seq_num 869) 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2824: delete oid 29 current snap is 276 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2819: finishing write tid 3 to smithi02683976-44 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 946 (ObjNum 885 snap 276 seq_num 885) dirty exists 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2819: left oid 44 (ObjNum 885 snap 276 seq_num 885) 2024-09-13T14:28:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:2818: expect (ObjNum 872 snap 269 seq_num 872) 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2820: finishing write tid 1 to smithi02683976-24 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2820: finishing write tid 2 to smithi02683976-24 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2818: done (4 left) 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2819: done (3 left) 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2824: done (2 left) 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 276 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2820: finishing write tid 3 to smithi02683976-24 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1263 (ObjNum 886 snap 276 seq_num 886) dirty exists 2024-09-13T14:28:35.227 INFO:tasks.rados.rados.0.smithi026.stdout:2820: left oid 24 (ObjNum 886 snap 276 seq_num 886) 2024-09-13T14:28:35.228 INFO:tasks.rados.rados.0.smithi026.stdout:2823: expect (ObjNum 869 snap 269 seq_num 869) 2024-09-13T14:28:35.312 INFO:tasks.rados.rados.0.smithi026.stdout:2825: seq_num 888 ranges {0=475136} 2024-09-13T14:28:35.315 INFO:tasks.rados.rados.0.smithi026.stdout:2825: writing smithi02683976-35 from 0 to 475136 tid 1 2024-09-13T14:28:35.315 INFO:tasks.rados.rados.0.smithi026.stdout:2820: done (2 left) 2024-09-13T14:28:35.315 INFO:tasks.rados.rados.0.smithi026.stdout:2823: done (1 left) 2024-09-13T14:28:35.315 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 276 2024-09-13T14:28:35.315 INFO:tasks.rados.rados.0.smithi026.stdout:2826: seq_num 889 ranges {417792=491520} 2024-09-13T14:28:35.320 INFO:tasks.rados.rados.0.smithi026.stdout:2826: writing smithi02683976-14 from 417792 to 909312 tid 1 2024-09-13T14:28:35.320 INFO:tasks.rados.rados.0.smithi026.stdout:2827: read oid 20 snap 247 2024-09-13T14:28:35.321 INFO:tasks.rados.rados.0.smithi026.stdout:2827: expect deleted 2024-09-13T14:28:35.321 INFO:tasks.rados.rados.0.smithi026.stdout:2828: snap_remove snap 266 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: pgmap v522: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Detected new or changed devices on smithi145 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.206 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all mds 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-13T14:28:36.207 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all rgw 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.208 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all nfs 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.209 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:36 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.233 INFO:tasks.rados.rados.0.smithi026.stdout:2825: finishing write tid 1 to smithi02683976-35 2024-09-13T14:28:36.233 INFO:tasks.rados.rados.0.smithi026.stdout:2825: finishing write tid 2 to smithi02683976-35 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2825: finishing write tid 3 to smithi02683976-35 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1579 (ObjNum 888 snap 276 seq_num 888) dirty exists 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2825: left oid 35 (ObjNum 888 snap 276 seq_num 888) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2826: finishing write tid 1 to smithi02683976-14 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2826: finishing write tid 2 to smithi02683976-14 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2826: finishing write tid 3 to smithi02683976-14 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1297 (ObjNum 889 snap 276 seq_num 889) dirty exists 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2826: left oid 14 (ObjNum 889 snap 276 seq_num 889) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2825: done (3 left) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2826: done (2 left) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2827: done (1 left) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2828: done (0 left) 2024-09-13T14:28:36.234 INFO:tasks.rados.rados.0.smithi026.stdout:2829: delete oid 35 current snap is 276 2024-09-13T14:28:36.246 INFO:tasks.rados.rados.0.smithi026.stdout:2829: done (0 left) 2024-09-13T14:28:36.246 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 276 2024-09-13T14:28:36.247 INFO:tasks.rados.rados.0.smithi026.stdout:2830: seq_num 891 ranges {425984=532480} 2024-09-13T14:28:36.252 INFO:tasks.rados.rados.0.smithi026.stdout:2830: writing smithi02683976-44 from 425984 to 958464 tid 1 2024-09-13T14:28:36.252 INFO:tasks.rados.rados.0.smithi026.stdout:2831: snap_create 2024-09-13T14:28:36.255 INFO:tasks.rados.rados.0.smithi026.stdout:2830: finishing write tid 1 to smithi02683976-44 2024-09-13T14:28:36.255 INFO:tasks.rados.rados.0.smithi026.stdout:2830: finishing write tid 2 to smithi02683976-44 2024-09-13T14:28:36.256 INFO:tasks.rados.rados.0.smithi026.stdout:2830: finishing write tid 3 to smithi02683976-44 2024-09-13T14:28:36.256 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 948 (ObjNum 891 snap 276 seq_num 891) dirty exists 2024-09-13T14:28:36.256 INFO:tasks.rados.rados.0.smithi026.stdout:2830: left oid 44 (ObjNum 891 snap 276 seq_num 891) 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: pgmap v522: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Detected new or changed devices on smithi145 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all mds 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all rgw 2024-09-13T14:28:36.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: pgmap v522: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 352 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:28:36.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Detected new or changed devices on smithi145 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all mds 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all rgw 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all nfs 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:28:36.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all nfs 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:28:36.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:36.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:36.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:36 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:37.236 INFO:tasks.rados.rados.0.smithi026.stdout:2830: done (1 left) 2024-09-13T14:28:37.237 INFO:tasks.rados.rados.0.smithi026.stdout:2831: done (0 left) 2024-09-13T14:28:37.237 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 277 2024-09-13T14:28:37.237 INFO:tasks.rados.rados.0.smithi026.stdout:2832: seq_num 892 ranges {6086656=548864} 2024-09-13T14:28:37.272 INFO:tasks.rados.rados.0.smithi026.stdout:2832: writing smithi02683976-13 from 6086656 to 6635520 tid 1 2024-09-13T14:28:37.273 INFO:tasks.rados.rados.0.smithi026.stdout:2833: copy_from oid 41 from oid 46 current snap is 277 2024-09-13T14:28:37.273 INFO:tasks.rados.rados.0.smithi026.stdout:2834: snap_remove snap 260 2024-09-13T14:28:37.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:37 smithi026 ceph-mon[97022]: osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:28:37.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:37 smithi026 ceph-mon[99461]: osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:28:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:37 smithi145 ceph-mon[89145]: osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:28:38.243 INFO:tasks.rados.rados.0.smithi026.stdout:2832: finishing write tid 1 to smithi02683976-13 2024-09-13T14:28:38.243 INFO:tasks.rados.rados.0.smithi026.stdout:2832: finishing write tid 2 to smithi02683976-13 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2832: finishing write tid 3 to smithi02683976-13 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1363 (ObjNum 892 snap 277 seq_num 892) dirty exists 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2832: left oid 13 (ObjNum 892 snap 277 seq_num 892) 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2833: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2833: finishing copy_from to smithi02683976-41 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 944 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2832: done (2 left) 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2833: done (1 left) 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2834: done (0 left) 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2835: read oid 34 snap -1 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2835: expect (ObjNum 884 snap 275 seq_num 884) 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:2836: rollback oid 50 current snap is 277 2024-09-13T14:28:38.244 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 275 2024-09-13T14:28:38.245 INFO:tasks.rados.rados.0.smithi026.stdout:2837: setattr oid 21 current snap is 277 2024-09-13T14:28:38.245 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 277 2024-09-13T14:28:38.245 INFO:tasks.rados.rados.0.smithi026.stdout:2838: seq_num 893 ranges {0=565248} 2024-09-13T14:28:38.249 INFO:tasks.rados.rados.0.smithi026.stdout:2838: writing smithi02683976-1 from 0 to 565248 tid 1 2024-09-13T14:28:38.249 INFO:tasks.rados.rados.0.smithi026.stdout:2839: snap_remove snap 249 2024-09-13T14:28:38.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[97022]: pgmap v525: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 368 KiB/s wr, 10 op/s 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[97022]: Upgrade: Updating grafana.a 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[97022]: Deploying daemon grafana.a on smithi145 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[97022]: osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[99461]: pgmap v525: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 368 KiB/s wr, 10 op/s 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[99461]: Upgrade: Updating grafana.a 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[99461]: Deploying daemon grafana.a on smithi145 2024-09-13T14:28:38.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:38 smithi026 ceph-mon[99461]: osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:28:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:38 smithi145 ceph-mon[89145]: pgmap v525: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 368 KiB/s wr, 10 op/s 2024-09-13T14:28:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:38 smithi145 ceph-mon[89145]: Upgrade: Updating grafana.a 2024-09-13T14:28:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:38 smithi145 ceph-mon[89145]: Deploying daemon grafana.a on smithi145 2024-09-13T14:28:38.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:38 smithi145 ceph-mon[89145]: osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:28:38.982 INFO:tasks.rados.rados.0.smithi026.stdout:2836: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:28:38.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-13T14:28:38.982 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1401 (ObjNum 1057 snap 0 seq_num 3890192592) dirty exists 2024-09-13T14:28:38.982 INFO:tasks.rados.rados.0.smithi026.stdout:2838: finishing write tid 1 to smithi02683976-1 2024-09-13T14:28:38.982 INFO:tasks.rados.rados.0.smithi026.stdout:2838: finishing write tid 2 to smithi02683976-1 2024-09-13T14:28:38.983 INFO:tasks.rados.rados.0.smithi026.stdout:2838: finishing write tid 3 to smithi02683976-1 2024-09-13T14:28:38.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1586 (ObjNum 893 snap 277 seq_num 893) dirty exists 2024-09-13T14:28:38.983 INFO:tasks.rados.rados.0.smithi026.stdout:2838: left oid 1 (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:28:38.983 INFO:tasks.rados.rados.0.smithi026.stdout:2835: expect (ObjNum 884 snap 275 seq_num 884) 2024-09-13T14:28:39.395 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:39 smithi145 ceph-mon[89145]: osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:28:39.395 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:39 smithi145 ceph-mon[89145]: osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:28:39.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:39 smithi026 ceph-mon[99461]: osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:28:39.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:39 smithi026 ceph-mon[99461]: osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:28:39.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:39 smithi026 ceph-mon[97022]: osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:28:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:39 smithi026 ceph-mon[97022]: osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2835: done (4 left) 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2836: done (3 left) 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2837: done (2 left) 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2838: done (1 left) 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2839: done (0 left) 2024-09-13T14:28:40.380 INFO:tasks.rados.rados.0.smithi026.stdout:2840: delete oid 35 current snap is 277 2024-09-13T14:28:40.382 INFO:tasks.rados.rados.0.smithi026.stdout:2840: done (0 left) 2024-09-13T14:28:40.382 INFO:tasks.rados.rados.0.smithi026.stdout:2841: snap_create 2024-09-13T14:28:40.475 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:40 smithi145 ceph-mon[89145]: pgmap v528: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 476 KiB/s wr, 16 op/s 2024-09-13T14:28:40.475 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:40.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:40 smithi026 ceph-mon[97022]: pgmap v528: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 476 KiB/s wr, 16 op/s 2024-09-13T14:28:40.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:40 smithi026 ceph-mon[99461]: pgmap v528: 121 pgs: 121 active+clean; 356 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 476 KiB/s wr, 16 op/s 2024-09-13T14:28:41.253 INFO:tasks.rados.rados.0.smithi026.stdout:2841: done (0 left) 2024-09-13T14:28:41.253 INFO:tasks.rados.rados.0.smithi026.stdout:2842: read oid 43 snap 271 2024-09-13T14:28:41.254 INFO:tasks.rados.rados.0.smithi026.stdout:2842: expect (ObjNum 845 snap 263 seq_num 845) 2024-09-13T14:28:41.254 INFO:tasks.rados.rados.0.smithi026.stdout:2843: snap_remove snap 257 2024-09-13T14:28:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:41.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:41] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:28:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:42.255 INFO:tasks.rados.rados.0.smithi026.stdout:2843: done (1 left) 2024-09-13T14:28:42.255 INFO:tasks.rados.rados.0.smithi026.stdout:2844: copy_from oid 44 from oid 27 current snap is 278 2024-09-13T14:28:42.255 INFO:tasks.rados.rados.0.smithi026.stdout:2842: expect (ObjNum 845 snap 263 seq_num 845) 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:2844: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:2844: finishing copy_from to smithi02683976-44 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1405 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:2842: done (1 left) 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:2844: done (0 left) 2024-09-13T14:28:42.401 INFO:tasks.rados.rados.0.smithi026.stdout:2845: delete oid 16 current snap is 278 2024-09-13T14:28:42.403 INFO:tasks.rados.rados.0.smithi026.stdout:2845: done (0 left) 2024-09-13T14:28:42.403 INFO:tasks.rados.rados.0.smithi026.stdout:2846: copy_from oid 6 from oid 12 current snap is 278 2024-09-13T14:28:42.403 INFO:tasks.rados.rados.0.smithi026.stdout:2847: copy_from oid 29 from oid 44 current snap is 278 2024-09-13T14:28:42.404 INFO:tasks.rados.rados.0.smithi026.stdout:2848: copy_from oid 14 from oid 11 current snap is 278 2024-09-13T14:28:42.404 INFO:tasks.rados.rados.0.smithi026.stdout:2849: delete oid 31 current snap is 278 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2846: finishing copy_from to smithi02683976-6 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2846: got expected ENOENT (src dne) 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2848: finishing copy_from to smithi02683976-14 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2848: got expected ENOENT (src dne) 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2846: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:28:42.406 INFO:tasks.rados.rados.0.smithi026.stdout:2848: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:28:42.407 INFO:tasks.rados.rados.0.smithi026.stdout:2846: done (3 left) 2024-09-13T14:28:42.407 INFO:tasks.rados.rados.0.smithi026.stdout:2848: done (2 left) 2024-09-13T14:28:42.407 INFO:tasks.rados.rados.0.smithi026.stdout:2849: done (1 left) 2024-09-13T14:28:42.408 INFO:tasks.rados.rados.0.smithi026.stdout:2850: copy_from oid 13 from oid 24 current snap is 278 2024-09-13T14:28:42.408 INFO:tasks.rados.rados.0.smithi026.stdout:2851: read oid 49 snap -1 2024-09-13T14:28:42.408 INFO:tasks.rados.rados.0.smithi026.stdout:2851: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:42.408 INFO:tasks.rados.rados.0.smithi026.stdout:2852: delete oid 7 current snap is 278 2024-09-13T14:28:42.410 INFO:tasks.rados.rados.0.smithi026.stdout:2852: done (3 left) 2024-09-13T14:28:42.410 INFO:tasks.rados.rados.0.smithi026.stdout:2853: rollback oid 19 current snap is 278 2024-09-13T14:28:42.410 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 269 2024-09-13T14:28:42.410 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 278 2024-09-13T14:28:42.410 INFO:tasks.rados.rados.0.smithi026.stdout:2854: seq_num 898 ranges {6463488=647168} 2024-09-13T14:28:42.450 INFO:tasks.rados.rados.0.smithi026.stdout:2854: writing smithi02683976-39 from 6463488 to 7110656 tid 1 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2855: read oid 2 snap 269 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2855: expect deleted 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 278 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2847: finishing copy_from racing read to smithi02683976-29 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2850: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2853: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2853: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1387 (ObjNum 802 snap 248 seq_num 802) dirty exists 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2850: finishing copy_from to smithi02683976-13 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1263 (ObjNum 886 snap 276 seq_num 886) dirty exists 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:2847: finishing copy_from to smithi02683976-29 2024-09-13T14:28:42.451 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1405 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:28:42.452 INFO:tasks.rados.rados.0.smithi026.stdout:2851: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:28:42.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:42 smithi026 ceph-mon[99461]: pgmap v530: 121 pgs: 121 active+clean; 355 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 710 KiB/s wr, 18 op/s 2024-09-13T14:28:42.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:42 smithi026 ceph-mon[99461]: osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:28:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:42 smithi026 ceph-mon[97022]: pgmap v530: 121 pgs: 121 active+clean; 355 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 710 KiB/s wr, 18 op/s 2024-09-13T14:28:42.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:42 smithi026 ceph-mon[97022]: osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:28:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:42 smithi145 ceph-mon[89145]: pgmap v530: 121 pgs: 121 active+clean; 355 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 710 KiB/s wr, 18 op/s 2024-09-13T14:28:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:42 smithi145 ceph-mon[89145]: osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:2854: finishing write tid 1 to smithi02683976-39 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:2854: finishing write tid 2 to smithi02683976-39 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:2854: finishing write tid 3 to smithi02683976-39 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1265 (ObjNum 898 snap 278 seq_num 898) dirty exists 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:2854: left oid 39 (ObjNum 898 snap 278 seq_num 898) 2024-09-13T14:28:42.728 INFO:tasks.rados.rados.0.smithi026.stdout:2856: seq_num 899 ranges {9904128=663552} 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2856: writing smithi02683976-3 from 9904128 to 10567680 tid 1 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2847: done (6 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2850: done (5 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2851: done (4 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2853: done (3 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2854: done (2 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2855: done (1 left) 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2857: copy_from oid 49 from oid 12 current snap is 278 2024-09-13T14:28:42.786 INFO:tasks.rados.rados.0.smithi026.stdout:2858: delete oid 14 current snap is 278 2024-09-13T14:28:42.787 INFO:tasks.rados.rados.0.smithi026.stdout:2857: finishing copy_from to smithi02683976-49 2024-09-13T14:28:42.787 INFO:tasks.rados.rados.0.smithi026.stdout:2857: got expected ENOENT (src dne) 2024-09-13T14:28:42.787 INFO:tasks.rados.rados.0.smithi026.stdout:2857: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:28:42.789 INFO:tasks.rados.rados.0.smithi026.stdout:2857: done (2 left) 2024-09-13T14:28:42.789 INFO:tasks.rados.rados.0.smithi026.stdout:2858: done (1 left) 2024-09-13T14:28:42.789 INFO:tasks.rados.rados.0.smithi026.stdout:2859: read oid 37 snap -1 2024-09-13T14:28:42.789 INFO:tasks.rados.rados.0.smithi026.stdout:2859: expect (ObjNum 738 snap 218 seq_num 738) 2024-09-13T14:28:42.789 INFO:tasks.rados.rados.0.smithi026.stdout:2860: rmattr oid 5 current snap is 278 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2860: done (2 left) 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2861: read oid 26 snap -1 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2861: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2862: read oid 16 snap -1 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2862: expect deleted 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2863: read oid 39 snap 263 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2863: expect deleted 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2864: read oid 14 snap -1 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2864: expect deleted 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2865: read oid 8 snap -1 2024-09-13T14:28:42.790 INFO:tasks.rados.rados.0.smithi026.stdout:2865: expect (ObjNum 860 snap 266 seq_num 860) 2024-09-13T14:28:42.791 INFO:tasks.rados.rados.0.smithi026.stdout:2866: read oid 1 snap 267 2024-09-13T14:28:42.791 INFO:tasks.rados.rados.0.smithi026.stdout:2866: expect deleted 2024-09-13T14:28:42.791 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 278 2024-09-13T14:28:42.791 INFO:tasks.rados.rados.0.smithi026.stdout:2867: seq_num 901 ranges {3121152=696320} 2024-09-13T14:28:42.810 INFO:tasks.rados.rados.0.smithi026.stdout:2867: writing smithi02683976-29 from 3121152 to 3817472 tid 1 2024-09-13T14:28:42.812 INFO:tasks.rados.rados.0.smithi026.stdout:2868: snap_remove snap 239 2024-09-13T14:28:43.268 INFO:tasks.rados.rados.0.smithi026.stdout:2868: done (9 left) 2024-09-13T14:28:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:2869: read oid 36 snap -1 2024-09-13T14:28:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:2869: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:28:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 278 2024-09-13T14:28:43.269 INFO:tasks.rados.rados.0.smithi026.stdout:2870: seq_num 902 ranges {12812288=712704} 2024-09-13T14:28:43.344 INFO:tasks.rados.rados.0.smithi026.stdout:2870: writing smithi02683976-48 from 12812288 to 13524992 tid 1 2024-09-13T14:28:43.344 INFO:tasks.rados.rados.0.smithi026.stdout:2871: snap_create 2024-09-13T14:28:43.344 INFO:tasks.rados.rados.0.smithi026.stdout:2859: expect (ObjNum 738 snap 218 seq_num 738) 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:2856: finishing write tid 1 to smithi02683976-3 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:2856: finishing write tid 2 to smithi02683976-3 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:2856: finishing write tid 3 to smithi02683976-3 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1622 (ObjNum 899 snap 278 seq_num 899) dirty exists 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:2856: left oid 3 (ObjNum 899 snap 278 seq_num 899) 2024-09-13T14:28:43.357 INFO:tasks.rados.rados.0.smithi026.stdout:2865: expect (ObjNum 860 snap 266 seq_num 860) 2024-09-13T14:28:43.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:43 smithi026 ceph-mon[99461]: osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:28:43.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:43 smithi026 ceph-mon[97022]: osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:28:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:43 smithi145 ceph-mon[89145]: osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:28:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:2867: finishing write tid 1 to smithi02683976-29 2024-09-13T14:28:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:2867: finishing write tid 2 to smithi02683976-29 2024-09-13T14:28:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:2867: finishing write tid 3 to smithi02683976-29 2024-09-13T14:28:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1407 (ObjNum 901 snap 278 seq_num 901) dirty exists 2024-09-13T14:28:44.239 INFO:tasks.rados.rados.0.smithi026.stdout:2867: left oid 29 (ObjNum 901 snap 278 seq_num 901) 2024-09-13T14:28:44.239 INFO:tasks.rados.rados.0.smithi026.stdout:2861: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:28:44.538 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:44 smithi145 ceph-mon[89145]: pgmap v533: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 734 KiB/s wr, 12 op/s 2024-09-13T14:28:44.538 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:44 smithi145 ceph-mon[89145]: osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:28:44.538 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:44 smithi145 ceph-mon[89145]: osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:28:44.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[99461]: pgmap v533: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 734 KiB/s wr, 12 op/s 2024-09-13T14:28:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[99461]: osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:28:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[99461]: osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:28:44.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[97022]: pgmap v533: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 734 KiB/s wr, 12 op/s 2024-09-13T14:28:44.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[97022]: osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:28:44.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:44 smithi026 ceph-mon[97022]: osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:28:45.882 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:28:45.913 INFO:tasks.rados.rados.0.smithi026.stdout:2869: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:28:46.081 INFO:tasks.rados.rados.0.smithi026.stdout:2870: finishing write tid 1 to smithi02683976-48 2024-09-13T14:28:46.081 INFO:tasks.rados.rados.0.smithi026.stdout:2870: finishing write tid 2 to smithi02683976-48 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2870: finishing write tid 3 to smithi02683976-48 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1150 (ObjNum 902 snap 278 seq_num 902) dirty exists 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2870: left oid 48 (ObjNum 902 snap 278 seq_num 902) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2856: done (11 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2859: done (10 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2861: done (9 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2862: done (8 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2863: done (7 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2864: done (6 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2865: done (5 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2866: done (4 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2867: done (3 left) 2024-09-13T14:28:46.082 INFO:tasks.rados.rados.0.smithi026.stdout:2869: done (2 left) 2024-09-13T14:28:46.083 INFO:tasks.rados.rados.0.smithi026.stdout:2870: done (1 left) 2024-09-13T14:28:46.083 INFO:tasks.rados.rados.0.smithi026.stdout:2871: done (0 left) 2024-09-13T14:28:46.083 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 279 2024-09-13T14:28:46.083 INFO:tasks.rados.rados.0.smithi026.stdout:2872: seq_num 903 ranges {3481600=729088} 2024-09-13T14:28:46.104 INFO:tasks.rados.rados.0.smithi026.stdout:2872: writing smithi02683976-25 from 3481600 to 4210688 tid 1 2024-09-13T14:28:46.104 INFO:tasks.rados.rados.0.smithi026.stdout:2873: rmattr oid 5 current snap is 279 2024-09-13T14:28:46.104 INFO:tasks.rados.rados.0.smithi026.stdout:2873: done (1 left) 2024-09-13T14:28:46.104 INFO:tasks.rados.rados.0.smithi026.stdout:2874: snap_create 2024-09-13T14:28:46.108 INFO:tasks.rados.rados.0.smithi026.stdout:2872: finishing write tid 1 to smithi02683976-25 2024-09-13T14:28:46.108 INFO:tasks.rados.rados.0.smithi026.stdout:2872: finishing write tid 2 to smithi02683976-25 2024-09-13T14:28:46.109 INFO:tasks.rados.rados.0.smithi026.stdout:2872: finishing write tid 3 to smithi02683976-25 2024-09-13T14:28:46.109 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1267 (ObjNum 903 snap 279 seq_num 903) dirty exists 2024-09-13T14:28:46.110 INFO:tasks.rados.rados.0.smithi026.stdout:2872: left oid 25 (ObjNum 903 snap 279 seq_num 903) 2024-09-13T14:28:46.273 INFO:tasks.rados.rados.0.smithi026.stdout:2872: done (1 left) 2024-09-13T14:28:46.273 INFO:tasks.rados.rados.0.smithi026.stdout:2874: done (0 left) 2024-09-13T14:28:46.273 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 280 2024-09-13T14:28:46.273 INFO:tasks.rados.rados.0.smithi026.stdout:2875: seq_num 904 ranges {1622016=745472} 2024-09-13T14:28:46.289 INFO:tasks.rados.rados.0.smithi026.stdout:2875: writing smithi02683976-40 from 1622016 to 2367488 tid 1 2024-09-13T14:28:46.289 INFO:tasks.rados.rados.0.smithi026.stdout:2876: snap_create 2024-09-13T14:28:46.294 INFO:tasks.rados.rados.0.smithi026.stdout:2875: finishing write tid 1 to smithi02683976-40 2024-09-13T14:28:46.294 INFO:tasks.rados.rados.0.smithi026.stdout:2875: finishing write tid 2 to smithi02683976-40 2024-09-13T14:28:46.295 INFO:tasks.rados.rados.0.smithi026.stdout:2875: finishing write tid 3 to smithi02683976-40 2024-09-13T14:28:46.295 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1184 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:28:46.295 INFO:tasks.rados.rados.0.smithi026.stdout:2875: left oid 40 (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:28:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:46 smithi145 ceph-mon[89145]: pgmap v536: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 134 KiB/s wr, 1 op/s 2024-09-13T14:28:46.474 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) 36s ago 12m 18.4M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (7m) 15s ago 12m 88.4M - 9.4.7 954c08fa6188 991d9256ad87 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (40s) 36s ago 13m 14.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 e163e8cfc713 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (19s) 15s ago 13m 11.8M - 19.3.0-4697-gd58f7306 fec07016a2b0 859c720bf426 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (6m) 15s ago 17m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) 36s ago 18m 529M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 36s ago 18m 66.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (5m) 15s ago 17m 51.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 36s ago 17m 55.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 36s ago 11m 11.1M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (7m) 15s ago 11m 10.7M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:28:46.475 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 36s ago 16m 242M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 36s ago 16m 240M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 36s ago 15m 193M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 36s ago 15m 96.1M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (2m) 15s ago 15m 163M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (2m) 15s ago 14m 145M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 b9422108344c 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (93s) 15s ago 14m 103M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 304ffd9a14d8 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (58s) 15s ago 13m 84.7M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 606a2c6e152f 2024-09-13T14:28:46.476 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (6m) 15s ago 12m 54.1M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:28:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:46 smithi026 ceph-mon[97022]: pgmap v536: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 134 KiB/s wr, 1 op/s 2024-09-13T14:28:46.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:46 smithi026 ceph-mon[99461]: pgmap v536: 121 pgs: 121 active+clean; 352 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 134 KiB/s wr, 1 op/s 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 8 2024-09-13T14:28:46.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 15 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:28:46.825 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "mds", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "osd", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "15/20 daemons upgraded", 2024-09-13T14:28:47.125 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T14:28:47.126 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-09-13T14:28:47.126 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:28:47.278 INFO:tasks.rados.rados.0.smithi026.stdout:2875: done (1 left) 2024-09-13T14:28:47.278 INFO:tasks.rados.rados.0.smithi026.stdout:2876: done (0 left) 2024-09-13T14:28:47.279 INFO:tasks.rados.rados.0.smithi026.stdout:2877: snap_create 2024-09-13T14:28:47.396 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:47 smithi145 ceph-mon[89145]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.397 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:47 smithi145 ceph-mon[89145]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.397 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:47 smithi145 ceph-mon[89145]: osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:28:47.397 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:47 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2808296174' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:47.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:47 smithi145 systemd[1]: Stopping Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[97022]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[97022]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[97022]: osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2808296174' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[99461]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[99461]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[99461]: osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:28:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:47 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2808296174' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:47.693 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:47 smithi145 podman[119319]: 2024-09-13 14:28:47.395786605 +0000 UTC m=+0.114931678 container died 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, name=ubi8, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.32.0, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, version=8.5, vcs-type=git, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8) 2024-09-13T14:28:47.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:47 smithi145 podman[119319]: 2024-09-13 14:28:47.774073081 +0000 UTC m=+0.493218142 container cleanup 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , version=8.5, description=Ceph Grafana Container, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.openshift.tags=base rhel8) 2024-09-13T14:28:47.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:47 smithi145 bash[119319]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:28:48.285 INFO:tasks.rados.rados.0.smithi026.stdout:2877: done (0 left) 2024-09-13T14:28:48.285 INFO:tasks.rados.rados.0.smithi026.stdout:2878: setattr oid 37 current snap is 282 2024-09-13T14:28:48.285 INFO:tasks.rados.rados.0.smithi026.stdout:2879: copy_from oid 13 from oid 1 current snap is 282 2024-09-13T14:28:48.285 INFO:tasks.rados.rados.0.smithi026.stdout:2880: delete oid 27 current snap is 282 2024-09-13T14:28:48.291 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1047 (ObjNum 738 snap 218 seq_num 738) dirty exists 2024-09-13T14:28:48.292 INFO:tasks.rados.rados.0.smithi026.stdout:2878: done (2 left) 2024-09-13T14:28:48.292 INFO:tasks.rados.rados.0.smithi026.stdout:2880: done (1 left) 2024-09-13T14:28:48.292 INFO:tasks.rados.rados.0.smithi026.stdout:2881: copy_from oid 18 from oid 34 current snap is 282 2024-09-13T14:28:48.292 INFO:tasks.rados.rados.0.smithi026.stdout:2882: snap_remove snap 241 2024-09-13T14:28:48.414 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:48 smithi145 ceph-mon[89145]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.415 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:48 smithi145 ceph-mon[89145]: pgmap v538: 121 pgs: 121 active+clean; 367 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:28:48.415 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:48 smithi145 ceph-mon[89145]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.415 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:48 smithi145 ceph-mon[89145]: osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:28:48.415 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 podman[119338]: 2024-09-13 14:28:48.050805865 +0000 UTC m=+0.653897818 container remove 991d9256ad8795436c7abee33d860fb042db2db795ff4fb149af84b9d960a79a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-13T14:28:48.415 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:28:48.415 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:28:48.415 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 systemd[1]: Stopped Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:48.415 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Consumed 7.121s CPU time. 2024-09-13T14:28:48.416 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 systemd[1]: Starting Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[97022]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[97022]: pgmap v538: 121 pgs: 121 active+clean; 367 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:28:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[97022]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[97022]: osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:28:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[99461]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[99461]: pgmap v538: 121 pgs: 121 active+clean; 367 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:28:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[99461]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:48 smithi026 ceph-mon[99461]: osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:28:48.705 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:48 smithi145 podman[119473]: 2024-09-13 14:28:48.526456997 +0000 UTC m=+0.016464474 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-13T14:28:48.982 INFO:tasks.rados.rados.0.smithi026.stdout:2879: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2879: finishing copy_from to smithi02683976-13 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1586 (ObjNum 893 snap 277 seq_num 893) dirty exists 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2881: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2881: finishing copy_from to smithi02683976-18 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 866 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2879: done (2 left) 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2881: done (1 left) 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2882: done (0 left) 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2883: rmattr oid 31 current snap is 282 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2883: done (0 left) 2024-09-13T14:28:48.983 INFO:tasks.rados.rados.0.smithi026.stdout:2884: setattr oid 17 current snap is 282 2024-09-13T14:28:48.984 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 282 2024-09-13T14:28:48.985 INFO:tasks.rados.rados.0.smithi026.stdout:2885: seq_num 906 ranges {1892352=778240} 2024-09-13T14:28:49.001 INFO:tasks.rados.rados.0.smithi026.stdout:2885: writing smithi02683976-47 from 1892352 to 2670592 tid 1 2024-09-13T14:28:49.002 INFO:tasks.rados.rados.0.smithi026.stdout:2886: read oid 43 snap -1 2024-09-13T14:28:49.002 INFO:tasks.rados.rados.0.smithi026.stdout:2886: expect deleted 2024-09-13T14:28:49.002 INFO:tasks.rados.rados.0.smithi026.stdout:2887: snap_remove snap 268 2024-09-13T14:28:49.614 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:49 smithi145 ceph-mon[89145]: osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:28:49.614 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:49 smithi145 ceph-mon[89145]: osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:28:49.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:49 smithi026 ceph-mon[97022]: osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:28:49.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:49 smithi026 ceph-mon[97022]: osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:28:49.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:49 smithi026 ceph-mon[99461]: osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:28:49.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:49 smithi026 ceph-mon[99461]: osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:28:49.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:49 smithi145 podman[119473]: 2024-09-13 14:28:49.614039009 +0000 UTC m=+1.104046485 container create 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:28:49.984 INFO:tasks.rados.rados.0.smithi026.stdout:2887: done (3 left) 2024-09-13T14:28:49.985 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 282 2024-09-13T14:28:49.985 INFO:tasks.rados.rados.0.smithi026.stdout:2888: seq_num 907 ranges {0=794624} 2024-09-13T14:28:49.990 INFO:tasks.rados.rados.0.smithi026.stdout:2888: writing smithi02683976-14 from 0 to 794624 tid 1 2024-09-13T14:28:49.991 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 945 (ObjNum 878 snap 273 seq_num 878) dirty exists 2024-09-13T14:28:49.991 INFO:tasks.rados.rados.0.smithi026.stdout:2885: finishing write tid 1 to smithi02683976-47 2024-09-13T14:28:49.991 INFO:tasks.rados.rados.0.smithi026.stdout:2885: finishing write tid 2 to smithi02683976-47 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2885: finishing write tid 3 to smithi02683976-47 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1152 (ObjNum 906 snap 282 seq_num 906) dirty exists 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2885: left oid 47 (ObjNum 906 snap 282 seq_num 906) 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2884: done (3 left) 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2885: done (2 left) 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2886: done (1 left) 2024-09-13T14:28:49.992 INFO:tasks.rados.rados.0.smithi026.stdout:2889: snap_remove snap 275 2024-09-13T14:28:50.287 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:28:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:50.564 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-mon[89145]: pgmap v541: 121 pgs: 121 active+clean; 368 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-13T14:28:50.564 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-mon[89145]: osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:28:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:50 smithi026 ceph-mon[97022]: pgmap v541: 121 pgs: 121 active+clean; 368 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-13T14:28:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:50 smithi026 ceph-mon[97022]: osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:28:50.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:50 smithi026 ceph-mon[99461]: pgmap v541: 121 pgs: 121 active+clean; 368 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-13T14:28:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:50 smithi026 ceph-mon[99461]: osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:28:50.826 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 podman[119473]: 2024-09-13 14:28:50.734303502 +0000 UTC m=+2.224310987 container init 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:28:50.826 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 podman[119473]: 2024-09-13 14:28:50.739592618 +0000 UTC m=+2.229600094 container start 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:28:50.988 INFO:tasks.rados.rados.0.smithi026.stdout:2889: done (1 left) 2024-09-13T14:28:50.988 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 282 2024-09-13T14:28:50.988 INFO:tasks.rados.rados.0.smithi026.stdout:2888: finishing write tid 1 to smithi02683976-14 2024-09-13T14:28:50.989 INFO:tasks.rados.rados.0.smithi026.stdout:2890: seq_num 908 ranges {7077888=409600,7487488=401408} 2024-09-13T14:28:51.030 INFO:tasks.rados.rados.0.smithi026.stdout:2890: writing smithi02683976-19 from 7077888 to 7487488 tid 1 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:2890: writing smithi02683976-19 from 7487488 to 7888896 tid 2 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:2891: snap_create 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:2888: finishing write tid 2 to smithi02683976-14 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:2888: finishing write tid 3 to smithi02683976-14 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1341 (ObjNum 907 snap 282 seq_num 907) dirty exists 2024-09-13T14:28:51.033 INFO:tasks.rados.rados.0.smithi026.stdout:2888: left oid 14 (ObjNum 907 snap 282 seq_num 907) 2024-09-13T14:28:51.037 INFO:tasks.rados.rados.0.smithi026.stdout:2890: finishing write tid 1 to smithi02683976-19 2024-09-13T14:28:51.037 INFO:tasks.rados.rados.0.smithi026.stdout:2890: finishing write tid 2 to smithi02683976-19 2024-09-13T14:28:51.037 INFO:tasks.rados.rados.0.smithi026.stdout:2890: finishing write tid 3 to smithi02683976-19 2024-09-13T14:28:51.039 INFO:tasks.rados.rados.0.smithi026.stdout:2890: finishing write tid 4 to smithi02683976-19 2024-09-13T14:28:51.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1589 (ObjNum 908 snap 282 seq_num 908) dirty exists 2024-09-13T14:28:51.039 INFO:tasks.rados.rados.0.smithi026.stdout:2890: left oid 19 (ObjNum 908 snap 282 seq_num 908) 2024-09-13T14:28:51.084 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 bash[119473]: 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 2024-09-13T14:28:51.084 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 systemd[1]: Started Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86858409Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-13T14:28:50Z 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86881227Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868820486Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868823776Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86882688Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868829468Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868832Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868834587Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868840339Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86884319Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86884573Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:28:51.085 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868848256Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:28:51.086 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868852939Z level=info msg=Target target=[all] 2024-09-13T14:28:51.086 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868858965Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:28:51.086 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868861528Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:28:51.086 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86886406Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:28:51.086 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.86886697Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868869478Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=settings t=2024-09-13T14:28:50.868872745Z level=info msg="App mode production" 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore t=2024-09-13T14:28:50.869114362Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore t=2024-09-13T14:28:50.869126761Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.86961861Z level=info msg="Starting DB migrations" 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.880734652Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.886446036Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.709873ms 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.886868617Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-13T14:28:51.087 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.887018214Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=149.733µs 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.887402473Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.888121782Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=718.88µs 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.890386972Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.895428594Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.040828ms 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.895985032Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.901301921Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.315994ms 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.901785541Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.9071547Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.3685ms 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.907511469Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-13T14:28:51.088 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.907628107Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=116.708µs 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.910200974Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.910953588Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=752.35µs 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.91305417Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.91378886Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=734.515µs 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.915664943Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.921064854Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.398245ms 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.921489453Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.92647987Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=4.987465ms 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.926902154Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.926952154Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=49.811µs 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.929572092Z level=info msg="Executing migration" id="permission kind migration" 2024-09-13T14:28:51.089 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.934762128Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.185083ms 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.935136801Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.940338455Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.195544ms 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.940828434Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.94674931Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.914275ms 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.947134202Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.947909039Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=774.561µs 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.949890182Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.950660546Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=770.052µs 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.952544164Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.953269749Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=725.488µs 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.95362256Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-13T14:28:51.090 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.95364174Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=19.374µs 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.956206011Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.956888606Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=681.944µs 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.958697975Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.959398933Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=699.774µs 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.96147859Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.976661117Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.180174ms 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.977157489Z level=info msg="Executing migration" id="create correlation v2" 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.977869463Z level=info msg="Migration successfully executed" id="create correlation v2" duration=711.593µs 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.97984617Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-13T14:28:51.091 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.98060418Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=759.242µs 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.982467752Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.983217111Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=749.094µs 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.983590146Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.984293137Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=702.819µs 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.984687366Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.984875001Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=187.561µs 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.9854446Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.986342258Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=896.594µs 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.986973844Z level=info msg="Executing migration" id="add provisioning column" 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.996909554Z level=info msg="Migration successfully executed" id="add provisioning column" duration=9.931276ms 2024-09-13T14:28:51.092 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.997799114Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.998110259Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=311.648µs 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.99876662Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:50.998901613Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=134.817µs 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.000380188Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.007402314Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=7.019291ms 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.007881114Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.008128648Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=246.306µs 2024-09-13T14:28:51.093 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.008561013Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.008685406Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=124.114µs 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.009058631Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.01004407Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=991.23µs 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.010407186Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.011133266Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=725.306µs 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.01147912Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-13T14:28:51.094 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.012263454Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=782.438µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.012614478Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.013002389Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=387.942µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.013474982Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.013672395Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=196.69µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.014281022Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.014980668Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=699.779µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.015334273Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.016068158Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=734.023µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.016755639Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.017415483Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=659.754µs 2024-09-13T14:28:51.095 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.019394581Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.020087732Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=697.961µs 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.022095031Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.02278378Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=688.732µs 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.024632327Z level=info msg="Executing migration" id="create anon_device table" 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.025208267Z level=info msg="Migration successfully executed" id="create anon_device table" duration=575.899µs 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.027309656Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.028008852Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=698.974µs 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.029996282Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.030674924Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=674.308µs 2024-09-13T14:28:51.096 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.031073238Z level=info msg="Executing migration" id="create signing_key table" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.031645033Z level=info msg="Migration successfully executed" id="create signing_key table" duration=571.853µs 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.032022431Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.032698965Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=675.482µs 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.033067768Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.033754724Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=687.146µs 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.034183952Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.034812819Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=631.1µs 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.035290423Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.041119834Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.82679ms 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.041518313Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.043292617Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.775043ms 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.043914137Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-13T14:28:51.097 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.044814619Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=900.209µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.045148898Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.045822707Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=673.629µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.046161406Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.046829113Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=667.189µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.047188529Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.048034928Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=845.938µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.048380069Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.049149223Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=768.662µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.049496322Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.050246575Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=749.934µs 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.050625062Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-13T14:28:51.098 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.051259171Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=634.149µs 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.051578099Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.051959799Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=379.933µs 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.052285922Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.052323758Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=39.367µs 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.053089016Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.058364422Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.27502ms 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.058739546Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.06402306Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.28316ms 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.06436705Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.064576548Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=209.545µs 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=migrator t=2024-09-13T14:28:51.065177448Z level=info msg="migrations completed" performed=65 skipped=482 duration=184.504882ms 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore t=2024-09-13T14:28:51.065822139Z level=info msg="Created default organization" 2024-09-13T14:28:51.099 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=secrets t=2024-09-13T14:28:51.067830151Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:28:51.100 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=plugin.store t=2024-09-13T14:28:51.083521995Z level=info msg="Loading plugins..." 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=local.finder t=2024-09-13T14:28:51.128054174Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=plugin.store t=2024-09-13T14:28:51.128080995Z level=info msg="Plugins loaded" count=55 duration=44.559275ms 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=query_data t=2024-09-13T14:28:51.130170636Z level=info msg="Query Service initialization" 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=live.push_http t=2024-09-13T14:28:51.132799051Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.migration t=2024-09-13T14:28:51.133932339Z level=info msg=Starting 2024-09-13T14:28:51.396 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.state.manager t=2024-09-13T14:28:51.13980516Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=infra.usagestats.collector t=2024-09-13T14:28:51.141319667Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.datasources t=2024-09-13T14:28:51.143937854Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.datasources t=2024-09-13T14:28:51.144238363Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.alerting t=2024-09-13T14:28:51.170955058Z level=info msg="starting to provision alerting" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.alerting t=2024-09-13T14:28:51.171050534Z level=info msg="finished to provision alerting" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafanaStorageLogger t=2024-09-13T14:28:51.17127225Z level=info msg="Storage starting" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.state.manager t=2024-09-13T14:28:51.171693492Z level=info msg="Warming state cache for startup" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:28:51.172114976Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=http.server t=2024-09-13T14:28:51.173975204Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=http.server t=2024-09-13T14:28:51.174452541Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.state.manager t=2024-09-13T14:28:51.191685143Z level=info msg="State cache has been initialized" states=0 duration=19.989325ms 2024-09-13T14:28:51.397 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ngalert.scheduler t=2024-09-13T14:28:51.191730642Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ticker t=2024-09-13T14:28:51.191783197Z level=info msg=starting first_tick=2024-09-13T14:29:00Z 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.dashboard t=2024-09-13T14:28:51.196011797Z level=info msg="starting to provision dashboards" 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=plugins.update.checker t=2024-09-13T14:28:51.253233995Z level=info msg="Update check succeeded" duration=82.007826ms 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafana.update.checker t=2024-09-13T14:28:51.255212349Z level=info msg="Update check succeeded" duration=83.185025ms 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore.transactions t=2024-09-13T14:28:51.268196286Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore.transactions t=2024-09-13T14:28:51.280144573Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:28:51.398 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=provisioning.dashboard t=2024-09-13T14:28:51.395264306Z level=info msg="finished to provision dashboards" 2024-09-13T14:28:51.647 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafana-apiserver t=2024-09-13T14:28:51.556936536Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-13T14:28:51.647 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafana-apiserver t=2024-09-13T14:28:51.557287465Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-13T14:28:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:28:51] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:28:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-mon[89145]: pgmap v544: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 380 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.7 MiB/s wr, 15 op/s 2024-09-13T14:28:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:51 smithi145 ceph-mon[89145]: osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2888: done (2 left) 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2890: done (1 left) 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2891: done (0 left) 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2892: read oid 30 snap -1 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2892: expect (ObjNum 855 snap 263 seq_num 855) 2024-09-13T14:28:51.990 INFO:tasks.rados.rados.0.smithi026.stdout:2893: delete oid 49 current snap is 283 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2893: done (1 left) 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2894: rollback oid 33 current snap is 283 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 280 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2895: copy_from oid 25 from oid 40 current snap is 283 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2896: read oid 8 snap -1 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2896: expect (ObjNum 860 snap 266 seq_num 860) 2024-09-13T14:28:51.993 INFO:tasks.rados.rados.0.smithi026.stdout:2897: snap_create 2024-09-13T14:28:52.000 INFO:tasks.rados.rados.0.smithi026.stdout:2894: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:28:52.002 INFO:tasks.rados.rados.0.smithi026.stdout:2894: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:28:52.002 INFO:tasks.rados.rados.0.smithi026.stdout:2894: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:28:52.003 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1660 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-13T14:28:52.007 INFO:tasks.rados.rados.0.smithi026.stdout:2892: expect (ObjNum 855 snap 263 seq_num 855) 2024-09-13T14:28:52.075 INFO:tasks.rados.rados.0.smithi026.stdout:2895: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:28:52.075 INFO:tasks.rados.rados.0.smithi026.stdout:2895: finishing copy_from to smithi02683976-25 2024-09-13T14:28:52.075 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1184 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:28:52.075 INFO:tasks.rados.rados.0.smithi026.stdout:2896: expect (ObjNum 860 snap 266 seq_num 860) 2024-09-13T14:28:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[99461]: pgmap v544: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 380 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.7 MiB/s wr, 15 op/s 2024-09-13T14:28:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:52.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[99461]: osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:28:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[97022]: pgmap v544: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 380 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.7 MiB/s wr, 15 op/s 2024-09-13T14:28:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:52.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:51 smithi026 ceph-mon[97022]: osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2892: done (4 left) 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2894: done (3 left) 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2895: done (2 left) 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2896: done (1 left) 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2897: done (0 left) 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2898: read oid 27 snap -1 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2898: expect deleted 2024-09-13T14:28:52.994 INFO:tasks.rados.rados.0.smithi026.stdout:2899: snap_remove snap 278 2024-09-13T14:28:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:52 smithi026 ceph-mon[97022]: osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:28:53.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:52 smithi026 ceph-mon[99461]: osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:28:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:52 smithi145 ceph-mon[89145]: osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:28:53.985 INFO:tasks.rados.rados.0.smithi026.stdout:2899: done (1 left) 2024-09-13T14:28:53.986 INFO:tasks.rados.rados.0.smithi026.stdout:2900: copy_from oid 5 from oid 47 current snap is 284 2024-09-13T14:28:53.986 INFO:tasks.rados.rados.0.smithi026.stdout:2898: done (1 left) 2024-09-13T14:28:53.986 INFO:tasks.rados.rados.0.smithi026.stdout:2901: snap_remove snap 248 2024-09-13T14:28:54.282 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:53 smithi145 ceph-mon[89145]: pgmap v547: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.1 MiB/s wr, 15 op/s 2024-09-13T14:28:54.283 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:53 smithi145 ceph-mon[89145]: osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:28:54.283 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:53 smithi145 ceph-mon[89145]: osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:28:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[99461]: pgmap v547: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.1 MiB/s wr, 15 op/s 2024-09-13T14:28:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[99461]: osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:28:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[99461]: osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:28:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[97022]: pgmap v547: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.1 MiB/s wr, 15 op/s 2024-09-13T14:28:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[97022]: osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:28:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-mon[97022]: osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:28:54.416 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:28:53 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:28:53.998+0000 7f450ab49640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 177 after trimming was started 2024-09-13T14:28:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:2900: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:28:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:2900: finishing copy_from to smithi02683976-5 2024-09-13T14:28:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1152 (ObjNum 906 snap 282 seq_num 906) dirty exists 2024-09-13T14:28:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:2900: done (1 left) 2024-09-13T14:28:54.986 INFO:tasks.rados.rados.0.smithi026.stdout:2901: done (0 left) 2024-09-13T14:28:54.987 INFO:tasks.rados.rados.0.smithi026.stdout:2902: read oid 11 snap 277 2024-09-13T14:28:54.987 INFO:tasks.rados.rados.0.smithi026.stdout:2902: expect deleted 2024-09-13T14:28:54.987 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 34 current snap is 284 2024-09-13T14:28:54.987 INFO:tasks.rados.rados.0.smithi026.stdout:2903: seq_num 910 ranges {11747328=442368,12189696=401408} 2024-09-13T14:28:55.055 INFO:tasks.rados.rados.0.smithi026.stdout:2903: writing smithi02683976-34 from 11747328 to 12189696 tid 1 2024-09-13T14:28:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2903: writing smithi02683976-34 from 12189696 to 12591104 tid 2 2024-09-13T14:28:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 284 2024-09-13T14:28:55.057 INFO:tasks.rados.rados.0.smithi026.stdout:2904: seq_num 911 ranges {13524992=466944,13991936=401408} 2024-09-13T14:28:55.133 INFO:tasks.rados.rados.0.smithi026.stdout:2904: writing smithi02683976-48 from 13524992 to 13991936 tid 1 2024-09-13T14:28:55.136 INFO:tasks.rados.rados.0.smithi026.stdout:2904: writing smithi02683976-48 from 13991936 to 14393344 tid 2 2024-09-13T14:28:55.136 INFO:tasks.rados.rados.0.smithi026.stdout:2905: copy_from oid 5 from oid 22 current snap is 284 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2903: finishing write tid 1 to smithi02683976-34 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2903: finishing write tid 2 to smithi02683976-34 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2903: finishing write tid 3 to smithi02683976-34 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2902: done (3 left) 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2906: read oid 26 snap -1 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2906: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2907: snap_create 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2903: finishing write tid 4 to smithi02683976-34 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 948 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-13T14:28:55.137 INFO:tasks.rados.rados.0.smithi026.stdout:2903: left oid 34 (ObjNum 910 snap 284 seq_num 910) 2024-09-13T14:28:55.138 INFO:tasks.rados.rados.0.smithi026.stdout:2904: finishing write tid 1 to smithi02683976-48 2024-09-13T14:28:55.140 INFO:tasks.rados.rados.0.smithi026.stdout:2904: finishing write tid 2 to smithi02683976-48 2024-09-13T14:28:55.140 INFO:tasks.rados.rados.0.smithi026.stdout:2904: finishing write tid 3 to smithi02683976-48 2024-09-13T14:28:55.141 INFO:tasks.rados.rados.0.smithi026.stdout:2904: finishing write tid 4 to smithi02683976-48 2024-09-13T14:28:55.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1155 (ObjNum 911 snap 284 seq_num 911) dirty exists 2024-09-13T14:28:55.141 INFO:tasks.rados.rados.0.smithi026.stdout:2904: left oid 48 (ObjNum 911 snap 284 seq_num 911) 2024-09-13T14:28:55.182 INFO:tasks.rados.rados.0.smithi026.stdout:2905: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:28:55.182 INFO:tasks.rados.rados.0.smithi026.stdout:2905: finishing copy_from to smithi02683976-5 2024-09-13T14:28:55.182 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1180 (ObjNum 607 snap 173 seq_num 607) dirty exists 2024-09-13T14:28:55.210 INFO:tasks.rados.rados.0.smithi026.stdout:2906: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: pgmap v550: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 716 KiB/s wr, 1 op/s 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.031 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:55 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: pgmap v550: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 716 KiB/s wr, 1 op/s 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: pgmap v550: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 383 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 716 KiB/s wr, 1 op/s 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:55 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:56.884 INFO:tasks.rados.rados.0.smithi026.stdout:2903: done (4 left) 2024-09-13T14:28:56.884 INFO:tasks.rados.rados.0.smithi026.stdout:2904: done (3 left) 2024-09-13T14:28:56.884 INFO:tasks.rados.rados.0.smithi026.stdout:2905: done (2 left) 2024-09-13T14:28:56.884 INFO:tasks.rados.rados.0.smithi026.stdout:2906: done (1 left) 2024-09-13T14:28:56.884 INFO:tasks.rados.rados.0.smithi026.stdout:2907: done (0 left) 2024-09-13T14:28:56.885 INFO:tasks.rados.rados.0.smithi026.stdout:2908: read oid 20 snap -1 2024-09-13T14:28:56.885 INFO:tasks.rados.rados.0.smithi026.stdout:2908: expect deleted 2024-09-13T14:28:56.885 INFO:tasks.rados.rados.0.smithi026.stdout:2909: snap_create 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2908: done (1 left) 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2909: done (0 left) 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2910: read oid 14 snap 276 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2910: expect (ObjNum 889 snap 276 seq_num 889) 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2911: rollback oid 10 current snap is 286 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 267 2024-09-13T14:28:56.991 INFO:tasks.rados.rados.0.smithi026.stdout:2912: delete oid 6 current snap is 286 2024-09-13T14:28:56.997 INFO:tasks.rados.rados.0.smithi026.stdout:2911: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:28:56.997 INFO:tasks.rados.rados.0.smithi026.stdout:2911: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:28:56.997 INFO:tasks.rados.rados.0.smithi026.stdout:2910: expect (ObjNum 889 snap 276 seq_num 889) 2024-09-13T14:28:57.029 INFO:tasks.rados.rados.0.smithi026.stdout:2911: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:28:57.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1188 (ObjNum 835 snap 260 seq_num 835) dirty exists 2024-09-13T14:28:57.030 INFO:tasks.rados.rados.0.smithi026.stdout:2910: done (2 left) 2024-09-13T14:28:57.030 INFO:tasks.rados.rados.0.smithi026.stdout:2911: done (1 left) 2024-09-13T14:28:57.030 INFO:tasks.rados.rados.0.smithi026.stdout:2912: done (0 left) 2024-09-13T14:28:57.030 INFO:tasks.rados.rados.0.smithi026.stdout:2913: snap_remove snap 277 2024-09-13T14:28:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:56 smithi026 ceph-mon[97022]: osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:28:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:56 smithi026 ceph-mon[99461]: osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:28:57.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:56 smithi145 ceph-mon[89145]: osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:28:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2913: done (0 left) 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2914: read oid 23 snap -1 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2914: expect (ObjNum 866 snap 266 seq_num 866) 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2915: rollback oid 35 current snap is 286 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 35 to 279 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2916: read oid 1 snap -1 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2916: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:28:58.001 INFO:tasks.rados.rados.0.smithi026.stdout:2917: snap_remove snap 280 2024-09-13T14:28:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:57 smithi145 ceph-mon[89145]: pgmap v553: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:28:58.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:57 smithi145 ceph-mon[89145]: osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:28:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:57 smithi026 ceph-mon[99461]: pgmap v553: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:28:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:57 smithi026 ceph-mon[99461]: osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:28:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:57 smithi026 ceph-mon[97022]: pgmap v553: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:28:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:57 smithi026 ceph-mon[97022]: osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:28:58.990 INFO:tasks.rados.rados.0.smithi026.stdout:2917: done (3 left) 2024-09-13T14:28:58.990 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 286 2024-09-13T14:28:58.990 INFO:tasks.rados.rados.0.smithi026.stdout:2918: seq_num 913 ranges {0=499712,499712=401408} 2024-09-13T14:28:58.994 INFO:tasks.rados.rados.0.smithi026.stdout:2918: writing smithi02683976-2 from 0 to 499712 tid 1 2024-09-13T14:28:58.996 INFO:tasks.rados.rados.0.smithi026.stdout:2918: writing smithi02683976-2 from 499712 to 901120 tid 2 2024-09-13T14:28:58.996 INFO:tasks.rados.rados.0.smithi026.stdout:2919: delete oid 22 current snap is 286 2024-09-13T14:28:58.997 INFO:tasks.rados.rados.0.smithi026.stdout:2915: finishing rollback tid 1 to smithi02683976-35 2024-09-13T14:28:58.998 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1580 (ObjNum 177 snap 0 seq_num 0) dirty dne 2024-09-13T14:28:58.998 INFO:tasks.rados.rados.0.smithi026.stdout:2916: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:28:59.015 INFO:tasks.rados.rados.0.smithi026.stdout:2914: expect (ObjNum 866 snap 266 seq_num 866) 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[97022]: osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:59.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: Upgrade: Finalizing container_image settings 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:28:59.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: Upgrade: Finalizing container_image settings 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:28:59.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:28:59.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:28:59.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:58 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: Upgrade: Finalizing container_image settings 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T14:28:59.458 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:28:59.459 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:59.460 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:59.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:59.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:28:59.461 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:58 smithi145 ceph-mon[89145]: osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:28:59.992 INFO:tasks.rados.rados.0.smithi026.stdout:2918: finishing write tid 1 to smithi02683976-2 2024-09-13T14:28:59.993 INFO:tasks.rados.rados.0.smithi026.stdout:2918: finishing write tid 2 to smithi02683976-2 2024-09-13T14:28:59.993 INFO:tasks.rados.rados.0.smithi026.stdout:2918: finishing write tid 3 to smithi02683976-2 2024-09-13T14:28:59.993 INFO:tasks.rados.rados.0.smithi026.stdout:2918: finishing write tid 4 to smithi02683976-2 2024-09-13T14:28:59.993 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1270 (ObjNum 913 snap 286 seq_num 913) dirty exists 2024-09-13T14:28:59.993 INFO:tasks.rados.rados.0.smithi026.stdout:2918: left oid 2 (ObjNum 913 snap 286 seq_num 913) 2024-09-13T14:28:59.996 INFO:tasks.rados.rados.0.smithi026.stdout:2914: done (4 left) 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2915: done (3 left) 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2916: done (2 left) 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2918: done (1 left) 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2919: done (0 left) 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2920: copy_from oid 37 from oid 15 current snap is 286 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 286 2024-09-13T14:28:59.997 INFO:tasks.rados.rados.0.smithi026.stdout:2921: seq_num 915 ranges {0=532480,532480=401408} 2024-09-13T14:28:59.999 INFO:tasks.rados.rados.0.smithi026.stdout:2921: writing smithi02683976-27 from 0 to 532480 tid 1 2024-09-13T14:29:00.002 INFO:tasks.rados.rados.0.smithi026.stdout:2921: writing smithi02683976-27 from 532480 to 933888 tid 2 2024-09-13T14:29:00.002 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 286 2024-09-13T14:29:00.003 INFO:tasks.rados.rados.0.smithi026.stdout:2922: seq_num 916 ranges {0=548864,548864=401408} 2024-09-13T14:29:00.008 INFO:tasks.rados.rados.0.smithi026.stdout:2922: writing smithi02683976-11 from 0 to 548864 tid 1 2024-09-13T14:29:00.010 INFO:tasks.rados.rados.0.smithi026.stdout:2922: writing smithi02683976-11 from 548864 to 950272 tid 2 2024-09-13T14:29:00.010 INFO:tasks.rados.rados.0.smithi026.stdout:2923: copy_from oid 4 from oid 16 current snap is 286 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2924: rollback oid 3 current snap is 286 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 276 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: finishing write tid 1 to smithi02683976-27 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2925: read oid 38 snap -1 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2925: expect deleted 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: finishing write tid 2 to smithi02683976-27 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: finishing write tid 3 to smithi02683976-27 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: finishing write tid 4 to smithi02683976-27 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1692 (ObjNum 915 snap 286 seq_num 915) dirty exists 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: left oid 27 (ObjNum 915 snap 286 seq_num 915) 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2921: done (5 left) 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:2926: rollback oid 19 current snap is 286 2024-09-13T14:29:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 282 2024-09-13T14:29:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:2927: copy_from oid 24 from oid 40 current snap is 286 2024-09-13T14:29:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 286 2024-09-13T14:29:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:2928: seq_num 917 ranges {11116544=565248,11681792=401408} 2024-09-13T14:29:00.075 INFO:tasks.rados.rados.0.smithi026.stdout:2928: writing smithi02683976-23 from 11116544 to 11681792 tid 1 2024-09-13T14:29:00.077 INFO:tasks.rados.rados.0.smithi026.stdout:2928: writing smithi02683976-23 from 11681792 to 12083200 tid 2 2024-09-13T14:29:00.077 INFO:tasks.rados.rados.0.smithi026.stdout:2929: snap_remove snap 284 2024-09-13T14:29:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: Upgrade: Complete! 2024-09-13T14:29:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[97022]: pgmap v556: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:29:00.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: Upgrade: Complete! 2024-09-13T14:29:00.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:28:59 smithi026 ceph-mon[99461]: pgmap v556: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:29:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:59 smithi145 ceph-mon[89145]: Upgrade: Complete! 2024-09-13T14:29:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:28:59 smithi145 ceph-mon[89145]: pgmap v556: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 387 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-13T14:29:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:01.009 INFO:tasks.rados.rados.0.smithi026.stdout:2929: done (8 left) 2024-09-13T14:29:01.009 INFO:tasks.rados.rados.0.smithi026.stdout:2930: rmattr oid 14 current snap is 286 2024-09-13T14:29:01.009 INFO:tasks.rados.rados.0.smithi026.stdout:2930: done (8 left) 2024-09-13T14:29:01.009 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 286 2024-09-13T14:29:01.009 INFO:tasks.rados.rados.0.smithi026.stdout:2931: seq_num 918 ranges {573440=581632,1155072=401408} 2024-09-13T14:29:01.016 INFO:tasks.rados.rados.0.smithi026.stdout:2931: writing smithi02683976-5 from 573440 to 1155072 tid 1 2024-09-13T14:29:01.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: writing smithi02683976-5 from 1155072 to 1556480 tid 2 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2922: finishing write tid 1 to smithi02683976-11 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2922: finishing write tid 2 to smithi02683976-11 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2920: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2924: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2922: finishing write tid 3 to smithi02683976-11 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:2920: finishing copy_from to smithi02683976-37 2024-09-13T14:29:01.020 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1280 (ObjNum 844 snap 263 seq_num 844) dirty exists 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2926: finishing rollback tid 0 to smithi02683976-19 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2923: finishing copy_from to smithi02683976-4 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2923: got expected ENOENT (src dne) 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2923: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2926: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2922: finishing write tid 4 to smithi02683976-11 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1455 (ObjNum 916 snap 286 seq_num 916) dirty exists 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2922: left oid 11 (ObjNum 916 snap 286 seq_num 916) 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2924: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:29:01.021 INFO:tasks.rados.rados.0.smithi026.stdout:2924: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1696 (ObjNum 879 snap 274 seq_num 879) dirty exists 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2926: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1592 (ObjNum 908 snap 282 seq_num 908) dirty exists 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2927: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2927: finishing copy_from to smithi02683976-24 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1184 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2928: finishing write tid 1 to smithi02683976-23 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2928: finishing write tid 2 to smithi02683976-23 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2928: finishing write tid 3 to smithi02683976-23 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2928: finishing write tid 4 to smithi02683976-23 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1369 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:01.022 INFO:tasks.rados.rados.0.smithi026.stdout:2928: left oid 23 (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2920: done (8 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2922: done (7 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2923: done (6 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2924: done (5 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2925: done (4 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2926: done (3 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2927: done (2 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2928: done (1 left) 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2932: read oid 43 snap -1 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2932: expect deleted 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2933: copy_from oid 30 from oid 28 current snap is 286 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2934: copy_from oid 14 from oid 34 current snap is 286 2024-09-13T14:29:01.023 INFO:tasks.rados.rados.0.smithi026.stdout:2935: snap_remove snap 281 2024-09-13T14:29:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:01] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:29:02.018 INFO:tasks.rados.rados.0.smithi026.stdout:2933: finishing copy_from racing read to smithi02683976-30 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2933: finishing copy_from to smithi02683976-30 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1180 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: finishing write tid 1 to smithi02683976-5 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: finishing write tid 2 to smithi02683976-5 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: finishing write tid 3 to smithi02683976-5 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: finishing write tid 4 to smithi02683976-5 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1702 (ObjNum 918 snap 286 seq_num 918) dirty exists 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: left oid 5 (ObjNum 918 snap 286 seq_num 918) 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2934: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2934: finishing copy_from to smithi02683976-14 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 948 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-13T14:29:02.019 INFO:tasks.rados.rados.0.smithi026.stdout:2931: done (4 left) 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2932: done (3 left) 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2933: done (2 left) 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2934: done (1 left) 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2935: done (0 left) 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2936: rmattr oid 40 current snap is 286 2024-09-13T14:29:02.020 INFO:tasks.rados.rados.0.smithi026.stdout:2937: snap_remove snap 273 2024-09-13T14:29:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[97022]: pgmap v558: 121 pgs: 121 active+clean; 390 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-13T14:29:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[97022]: osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:29:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:29:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[99461]: pgmap v558: 121 pgs: 121 active+clean; 390 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-13T14:29:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[99461]: osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:29:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:29:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:02 smithi145 ceph-mon[89145]: pgmap v558: 121 pgs: 121 active+clean; 390 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-13T14:29:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:02 smithi145 ceph-mon[89145]: osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:29:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:29:03.021 INFO:tasks.rados.rados.0.smithi026.stdout:2937: done (1 left) 2024-09-13T14:29:03.021 INFO:tasks.rados.rados.0.smithi026.stdout:2938: rollback oid 44 current snap is 286 2024-09-13T14:29:03.022 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 271 2024-09-13T14:29:03.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1189 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:03.022 INFO:tasks.rados.rados.0.smithi026.stdout:2936: done (1 left) 2024-09-13T14:29:03.022 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 286 2024-09-13T14:29:03.022 INFO:tasks.rados.rados.0.smithi026.stdout:2939: seq_num 919 ranges {1703936=598016,2301952=401408} 2024-09-13T14:29:03.035 INFO:tasks.rados.rados.0.smithi026.stdout:2939: writing smithi02683976-4 from 1703936 to 2301952 tid 1 2024-09-13T14:29:03.041 INFO:tasks.rados.rados.0.smithi026.stdout:2939: writing smithi02683976-4 from 2301952 to 2703360 tid 2 2024-09-13T14:29:03.041 INFO:tasks.rados.rados.0.smithi026.stdout:2940: read oid 19 snap -1 2024-09-13T14:29:03.041 INFO:tasks.rados.rados.0.smithi026.stdout:2940: expect (ObjNum 908 snap 282 seq_num 908) 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2938: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2938: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1407 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2938: done (2 left) 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2941: read oid 34 snap -1 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2941: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2942: rollback oid 30 current snap is 286 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 264 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2943: read oid 49 snap -1 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2943: expect deleted 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 286 2024-09-13T14:29:03.042 INFO:tasks.rados.rados.0.smithi026.stdout:2944: seq_num 920 ranges {9199616=614400,9814016=401408} 2024-09-13T14:29:03.095 INFO:tasks.rados.rados.0.smithi026.stdout:2944: writing smithi02683976-8 from 9199616 to 9814016 tid 1 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2944: writing smithi02683976-8 from 9814016 to 10215424 tid 2 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2945: rollback oid 46 current snap is 286 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 276 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2939: finishing write tid 1 to smithi02683976-4 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2939: finishing write tid 2 to smithi02683976-4 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2943: done (6 left) 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2946: rollback oid 13 current snap is 286 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 270 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2939: finishing write tid 3 to smithi02683976-4 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2942: finishing rollback tid 0 to smithi02683976-30 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2942: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:2942: finishing rollback tid 2 to smithi02683976-30 2024-09-13T14:29:03.098 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1298 (ObjNum 855 snap 263 seq_num 855) dirty exists 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2939: finishing write tid 4 to smithi02683976-4 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1295 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2939: left oid 4 (ObjNum 919 snap 286 seq_num 919) 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2939: done (6 left) 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2942: done (5 left) 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2947: rmattr oid 41 current snap is 286 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2947: done (5 left) 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2948: copy_from oid 3 from oid 48 current snap is 286 2024-09-13T14:29:03.099 INFO:tasks.rados.rados.0.smithi026.stdout:2940: expect (ObjNum 908 snap 282 seq_num 908) 2024-09-13T14:29:03.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:03 smithi026 ceph-mon[99461]: osdmap e639: 8 total, 8 up, 8 in 2024-09-13T14:29:03.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:03 smithi026 ceph-mon[97022]: osdmap e639: 8 total, 8 up, 8 in 2024-09-13T14:29:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:03 smithi145 ceph-mon[89145]: osdmap e639: 8 total, 8 up, 8 in 2024-09-13T14:29:03.800 INFO:tasks.rados.rados.0.smithi026.stdout:2944: finishing write tid 1 to smithi02683976-8 2024-09-13T14:29:03.800 INFO:tasks.rados.rados.0.smithi026.stdout:2945: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2944: finishing write tid 2 to smithi02683976-8 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2940: done (5 left) 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2949: read oid 6 snap -1 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2949: expect deleted 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2950: copy_from oid 21 from oid 47 current snap is 286 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2945: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2945: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1410 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2944: finishing write tid 3 to smithi02683976-8 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2946: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2945: done (6 left) 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2951: copy_from oid 22 from oid 18 current snap is 286 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:2944: finishing write tid 4 to smithi02683976-8 2024-09-13T14:29:03.801 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1285 (ObjNum 920 snap 286 seq_num 920) dirty exists 2024-09-13T14:29:03.802 INFO:tasks.rados.rados.0.smithi026.stdout:2944: left oid 8 (ObjNum 920 snap 286 seq_num 920) 2024-09-13T14:29:03.802 INFO:tasks.rados.rados.0.smithi026.stdout:2946: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:29:03.802 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1594 (ObjNum 65 snap 0 seq_num 1599885155) dirty dne 2024-09-13T14:29:03.802 INFO:tasks.rados.rados.0.smithi026.stdout:2941: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[97022]: pgmap v561: 121 pgs: 121 active+clean; 392 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[97022]: osdmap e640: 8 total, 8 up, 8 in 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[97022]: osdmap e641: 8 total, 8 up, 8 in 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[99461]: pgmap v561: 121 pgs: 121 active+clean; 392 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[99461]: osdmap e640: 8 total, 8 up, 8 in 2024-09-13T14:29:04.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:04 smithi026 ceph-mon[99461]: osdmap e641: 8 total, 8 up, 8 in 2024-09-13T14:29:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:04 smithi145 ceph-mon[89145]: pgmap v561: 121 pgs: 121 active+clean; 392 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-13T14:29:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:04 smithi145 ceph-mon[89145]: osdmap e640: 8 total, 8 up, 8 in 2024-09-13T14:29:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:04 smithi145 ceph-mon[89145]: osdmap e641: 8 total, 8 up, 8 in 2024-09-13T14:29:05.407 INFO:tasks.rados.rados.0.smithi026.stdout:2948: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:29:05.407 INFO:tasks.rados.rados.0.smithi026.stdout:2948: finishing copy_from to smithi02683976-3 2024-09-13T14:29:05.407 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1155 (ObjNum 911 snap 284 seq_num 911) dirty exists 2024-09-13T14:29:05.407 INFO:tasks.rados.rados.0.smithi026.stdout:2941: done (6 left) 2024-09-13T14:29:05.407 INFO:tasks.rados.rados.0.smithi026.stdout:2944: done (5 left) 2024-09-13T14:29:05.408 INFO:tasks.rados.rados.0.smithi026.stdout:2946: done (4 left) 2024-09-13T14:29:05.408 INFO:tasks.rados.rados.0.smithi026.stdout:2948: done (3 left) 2024-09-13T14:29:05.408 INFO:tasks.rados.rados.0.smithi026.stdout:2952: snap_remove snap 270 2024-09-13T14:29:06.028 INFO:tasks.rados.rados.0.smithi026.stdout:2952: done (3 left) 2024-09-13T14:29:06.028 INFO:tasks.rados.rados.0.smithi026.stdout:2953: read oid 9 snap -1 2024-09-13T14:29:06.028 INFO:tasks.rados.rados.0.smithi026.stdout:2953: expect (ObjNum 836 snap 261 seq_num 836) 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2954: copy_from oid 34 from oid 26 current snap is 286 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2955: snap_create 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2950: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2950: finishing copy_from to smithi02683976-21 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1152 (ObjNum 906 snap 282 seq_num 906) dirty exists 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2951: finishing copy_from racing read to smithi02683976-22 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:2951: finishing copy_from to smithi02683976-22 2024-09-13T14:29:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 866 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:29:06.060 INFO:tasks.rados.rados.0.smithi026.stdout:2953: expect (ObjNum 836 snap 261 seq_num 836) 2024-09-13T14:29:06.192 INFO:tasks.rados.rados.0.smithi026.stdout:2954: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:29:06.192 INFO:tasks.rados.rados.0.smithi026.stdout:2954: finishing copy_from to smithi02683976-34 2024-09-13T14:29:06.192 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1193 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:29:06.271 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:06 smithi145 ceph-mon[89145]: pgmap v564: 121 pgs: 121 active+clean; 396 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-13T14:29:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:06 smithi026 ceph-mon[97022]: pgmap v564: 121 pgs: 121 active+clean; 396 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-13T14:29:06.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:06 smithi026 ceph-mon[99461]: pgmap v564: 121 pgs: 121 active+clean; 396 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-13T14:29:07.033 INFO:tasks.rados.rados.0.smithi026.stdout:2949: done (5 left) 2024-09-13T14:29:07.033 INFO:tasks.rados.rados.0.smithi026.stdout:2950: done (4 left) 2024-09-13T14:29:07.033 INFO:tasks.rados.rados.0.smithi026.stdout:2951: done (3 left) 2024-09-13T14:29:07.033 INFO:tasks.rados.rados.0.smithi026.stdout:2953: done (2 left) 2024-09-13T14:29:07.034 INFO:tasks.rados.rados.0.smithi026.stdout:2954: done (1 left) 2024-09-13T14:29:07.034 INFO:tasks.rados.rados.0.smithi026.stdout:2955: done (0 left) 2024-09-13T14:29:07.034 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 287 2024-09-13T14:29:07.034 INFO:tasks.rados.rados.0.smithi026.stdout:2956: seq_num 921 ranges {2670592=630784,3301376=401408} 2024-09-13T14:29:07.051 INFO:tasks.rados.rados.0.smithi026.stdout:2956: writing smithi02683976-47 from 2670592 to 3301376 tid 1 2024-09-13T14:29:07.053 INFO:tasks.rados.rados.0.smithi026.stdout:2956: writing smithi02683976-47 from 3301376 to 3702784 tid 2 2024-09-13T14:29:07.054 INFO:tasks.rados.rados.0.smithi026.stdout:2957: snap_remove snap 274 2024-09-13T14:29:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:07 smithi026 ceph-mon[97022]: osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:29:07.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:07 smithi026 ceph-mon[99461]: osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:29:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:07 smithi145 ceph-mon[89145]: osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2957: done (1 left) 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2958: read oid 5 snap -1 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2958: expect (ObjNum 918 snap 286 seq_num 918) 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2959: copy_from oid 9 from oid 25 current snap is 287 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2956: finishing write tid 1 to smithi02683976-47 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2956: finishing write tid 2 to smithi02683976-47 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2956: finishing write tid 3 to smithi02683976-47 2024-09-13T14:29:08.039 INFO:tasks.rados.rados.0.smithi026.stdout:2956: finishing write tid 4 to smithi02683976-47 2024-09-13T14:29:08.040 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1158 (ObjNum 921 snap 287 seq_num 921) dirty exists 2024-09-13T14:29:08.040 INFO:tasks.rados.rados.0.smithi026.stdout:2956: left oid 47 (ObjNum 921 snap 287 seq_num 921) 2024-09-13T14:29:08.040 INFO:tasks.rados.rados.0.smithi026.stdout:2956: done (2 left) 2024-09-13T14:29:08.040 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 287 2024-09-13T14:29:08.040 INFO:tasks.rados.rados.0.smithi026.stdout:2960: seq_num 922 ranges {0=647168,647168=401408} 2024-09-13T14:29:08.043 INFO:tasks.rados.rados.0.smithi026.stdout:2960: writing smithi02683976-49 from 0 to 647168 tid 1 2024-09-13T14:29:08.047 INFO:tasks.rados.rados.0.smithi026.stdout:2960: writing smithi02683976-49 from 647168 to 1048576 tid 2 2024-09-13T14:29:08.048 INFO:tasks.rados.rados.0.smithi026.stdout:2961: read oid 7 snap -1 2024-09-13T14:29:08.048 INFO:tasks.rados.rados.0.smithi026.stdout:2961: expect deleted 2024-09-13T14:29:08.048 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 287 2024-09-13T14:29:08.048 INFO:tasks.rados.rados.0.smithi026.stdout:2962: seq_num 923 ranges {376832=663552,1040384=401408} 2024-09-13T14:29:08.054 INFO:tasks.rados.rados.0.smithi026.stdout:2962: writing smithi02683976-46 from 376832 to 1040384 tid 1 2024-09-13T14:29:08.057 INFO:tasks.rados.rados.0.smithi026.stdout:2962: writing smithi02683976-46 from 1040384 to 1441792 tid 2 2024-09-13T14:29:08.058 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 287 2024-09-13T14:29:08.058 INFO:tasks.rados.rados.0.smithi026.stdout:2963: seq_num 924 ranges {2703360=679936,3383296=401408} 2024-09-13T14:29:08.077 INFO:tasks.rados.rados.0.smithi026.stdout:2963: writing smithi02683976-4 from 2703360 to 3383296 tid 1 2024-09-13T14:29:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:2963: writing smithi02683976-4 from 3383296 to 3784704 tid 2 2024-09-13T14:29:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:2964: rmattr oid 2 current snap is 287 2024-09-13T14:29:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:2964: done (6 left) 2024-09-13T14:29:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 287 2024-09-13T14:29:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:2965: seq_num 925 ranges {0=696320,696320=401408} 2024-09-13T14:29:08.083 INFO:tasks.rados.rados.0.smithi026.stdout:2965: writing smithi02683976-12 from 0 to 696320 tid 1 2024-09-13T14:29:08.086 INFO:tasks.rados.rados.0.smithi026.stdout:2965: writing smithi02683976-12 from 696320 to 1097728 tid 2 2024-09-13T14:29:08.086 INFO:tasks.rados.rados.0.smithi026.stdout:2966: setattr oid 47 current snap is 287 2024-09-13T14:29:08.086 INFO:tasks.rados.rados.0.smithi026.stdout:2958: expect (ObjNum 918 snap 286 seq_num 918) 2024-09-13T14:29:08.140 INFO:tasks.rados.rados.0.smithi026.stdout:2960: finishing write tid 1 to smithi02683976-49 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2960: finishing write tid 2 to smithi02683976-49 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2960: finishing write tid 3 to smithi02683976-49 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2962: finishing write tid 1 to smithi02683976-46 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2960: finishing write tid 4 to smithi02683976-49 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1192 (ObjNum 922 snap 287 seq_num 922) dirty exists 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2960: left oid 49 (ObjNum 922 snap 287 seq_num 922) 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2959: finishing copy_from racing read to smithi02683976-9 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2962: finishing write tid 2 to smithi02683976-46 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2962: finishing write tid 3 to smithi02683976-46 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2962: finishing write tid 4 to smithi02683976-46 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1413 (ObjNum 923 snap 287 seq_num 923) dirty exists 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2962: left oid 46 (ObjNum 923 snap 287 seq_num 923) 2024-09-13T14:29:08.141 INFO:tasks.rados.rados.0.smithi026.stdout:2959: finishing copy_from to smithi02683976-9 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1184 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2963: finishing write tid 1 to smithi02683976-4 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2963: finishing write tid 2 to smithi02683976-4 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2963: finishing write tid 3 to smithi02683976-4 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2963: finishing write tid 4 to smithi02683976-4 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1301 (ObjNum 924 snap 287 seq_num 924) dirty exists 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2963: left oid 4 (ObjNum 924 snap 287 seq_num 924) 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2965: finishing write tid 1 to smithi02683976-12 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2965: finishing write tid 2 to smithi02683976-12 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2965: finishing write tid 3 to smithi02683976-12 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2965: finishing write tid 4 to smithi02683976-12 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1197 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-13T14:29:08.142 INFO:tasks.rados.rados.0.smithi026.stdout:2965: left oid 12 (ObjNum 925 snap 287 seq_num 925) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2958: done (7 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2959: done (6 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2960: done (5 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2961: done (4 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2962: done (3 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2963: done (2 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2965: done (1 left) 2024-09-13T14:29:08.143 INFO:tasks.rados.rados.0.smithi026.stdout:2967: snap_remove snap 286 2024-09-13T14:29:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:08 smithi026 ceph-mon[97022]: pgmap v566: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 438 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.7 MiB/s wr, 11 op/s 2024-09-13T14:29:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:08 smithi026 ceph-mon[97022]: osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:29:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:08 smithi026 ceph-mon[99461]: pgmap v566: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 438 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.7 MiB/s wr, 11 op/s 2024-09-13T14:29:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:08 smithi026 ceph-mon[99461]: osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:29:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:08 smithi145 ceph-mon[89145]: pgmap v566: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 438 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.7 MiB/s wr, 11 op/s 2024-09-13T14:29:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:08 smithi145 ceph-mon[89145]: osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:29:08.989 INFO:tasks.rados.rados.0.smithi026.stdout:2967: done (1 left) 2024-09-13T14:29:08.989 INFO:tasks.rados.rados.0.smithi026.stdout:2968: snap_remove snap 255 2024-09-13T14:29:08.989 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1159 (ObjNum 921 snap 287 seq_num 921) dirty exists 2024-09-13T14:29:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:09 smithi026 ceph-mon[97022]: osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:29:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:09 smithi026 ceph-mon[97022]: osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:29:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:09 smithi026 ceph-mon[99461]: osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:29:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:09 smithi026 ceph-mon[99461]: osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:29:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:09 smithi145 ceph-mon[89145]: osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:29:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:09 smithi145 ceph-mon[89145]: osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:29:09.991 INFO:tasks.rados.rados.0.smithi026.stdout:2966: done (1 left) 2024-09-13T14:29:09.991 INFO:tasks.rados.rados.0.smithi026.stdout:2968: done (0 left) 2024-09-13T14:29:09.991 INFO:tasks.rados.rados.0.smithi026.stdout:2969: copy_from oid 32 from oid 23 current snap is 287 2024-09-13T14:29:09.991 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 287 2024-09-13T14:29:09.991 INFO:tasks.rados.rados.0.smithi026.stdout:2970: seq_num 926 ranges {0=712704,712704=401408} 2024-09-13T14:29:09.996 INFO:tasks.rados.rados.0.smithi026.stdout:2970: writing smithi02683976-38 from 0 to 712704 tid 1 2024-09-13T14:29:09.999 INFO:tasks.rados.rados.0.smithi026.stdout:2970: writing smithi02683976-38 from 712704 to 1114112 tid 2 2024-09-13T14:29:10.000 INFO:tasks.rados.rados.0.smithi026.stdout:2971: snap_create 2024-09-13T14:29:10.014 INFO:tasks.rados.rados.0.smithi026.stdout:2970: finishing write tid 1 to smithi02683976-38 2024-09-13T14:29:10.022 INFO:tasks.rados.rados.0.smithi026.stdout:2970: finishing write tid 2 to smithi02683976-38 2024-09-13T14:29:10.022 INFO:tasks.rados.rados.0.smithi026.stdout:2970: finishing write tid 3 to smithi02683976-38 2024-09-13T14:29:10.024 INFO:tasks.rados.rados.0.smithi026.stdout:2970: finishing write tid 4 to smithi02683976-38 2024-09-13T14:29:10.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1398 (ObjNum 926 snap 287 seq_num 926) dirty exists 2024-09-13T14:29:10.024 INFO:tasks.rados.rados.0.smithi026.stdout:2970: left oid 38 (ObjNum 926 snap 287 seq_num 926) 2024-09-13T14:29:10.057 INFO:tasks.rados.rados.0.smithi026.stdout:2969: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:29:10.059 INFO:tasks.rados.rados.0.smithi026.stdout:2969: finishing copy_from to smithi02683976-32 2024-09-13T14:29:10.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1369 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:10 smithi026 ceph-mon[97022]: pgmap v569: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 439 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.2 MiB/s wr, 11 op/s 2024-09-13T14:29:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:10 smithi026 ceph-mon[97022]: osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:29:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:10 smithi026 ceph-mon[99461]: pgmap v569: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 439 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.2 MiB/s wr, 11 op/s 2024-09-13T14:29:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:10 smithi026 ceph-mon[99461]: osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:29:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:10 smithi145 ceph-mon[89145]: pgmap v569: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 439 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 9.2 MiB/s wr, 11 op/s 2024-09-13T14:29:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:10 smithi145 ceph-mon[89145]: osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:29:10.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:10.991 INFO:tasks.rados.rados.0.smithi026.stdout:2969: done (2 left) 2024-09-13T14:29:10.992 INFO:tasks.rados.rados.0.smithi026.stdout:2970: done (1 left) 2024-09-13T14:29:10.992 INFO:tasks.rados.rados.0.smithi026.stdout:2971: done (0 left) 2024-09-13T14:29:10.992 INFO:tasks.rados.rados.0.smithi026.stdout:2972: snap_remove snap 287 2024-09-13T14:29:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:11] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.51.0" 2024-09-13T14:29:11.997 INFO:tasks.rados.rados.0.smithi026.stdout:2972: done (0 left) 2024-09-13T14:29:11.997 INFO:tasks.rados.rados.0.smithi026.stdout:2973: copy_from oid 16 from oid 21 current snap is 288 2024-09-13T14:29:11.997 INFO:tasks.rados.rados.0.smithi026.stdout:2974: rmattr oid 13 current snap is 288 2024-09-13T14:29:11.997 INFO:tasks.rados.rados.0.smithi026.stdout:2974: done (1 left) 2024-09-13T14:29:11.998 INFO:tasks.rados.rados.0.smithi026.stdout:2975: delete oid 33 current snap is 288 2024-09-13T14:29:12.002 INFO:tasks.rados.rados.0.smithi026.stdout:2975: done (1 left) 2024-09-13T14:29:12.002 INFO:tasks.rados.rados.0.smithi026.stdout:2976: snap_remove snap 267 2024-09-13T14:29:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[99461]: pgmap v572: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 460 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 14 op/s 2024-09-13T14:29:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[99461]: osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:29:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[97022]: pgmap v572: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 460 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 14 op/s 2024-09-13T14:29:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[97022]: osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:29:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:11 smithi145 ceph-mon[89145]: pgmap v572: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 460 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 14 op/s 2024-09-13T14:29:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:11 smithi145 ceph-mon[89145]: osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:29:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:13.004 INFO:tasks.rados.rados.0.smithi026.stdout:2976: done (1 left) 2024-09-13T14:29:13.004 INFO:tasks.rados.rados.0.smithi026.stdout:2977: rollback oid 50 current snap is 288 2024-09-13T14:29:13.004 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 285 2024-09-13T14:29:13.004 INFO:tasks.rados.rados.0.smithi026.stdout:2973: finishing copy_from racing read to smithi02683976-16 2024-09-13T14:29:13.005 INFO:tasks.rados.rados.0.smithi026.stdout:2973: finishing copy_from to smithi02683976-16 2024-09-13T14:29:13.005 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1152 (ObjNum 906 snap 282 seq_num 906) dirty exists 2024-09-13T14:29:13.005 INFO:tasks.rados.rados.0.smithi026.stdout:2973: done (1 left) 2024-09-13T14:29:13.005 INFO:tasks.rados.rados.0.smithi026.stdout:2978: setattr oid 46 current snap is 288 2024-09-13T14:29:13.007 INFO:tasks.rados.rados.0.smithi026.stdout:2979: copy_from oid 15 from oid 33 current snap is 288 2024-09-13T14:29:13.008 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 288 2024-09-13T14:29:13.008 INFO:tasks.rados.rados.0.smithi026.stdout:2980: seq_num 928 ranges {0=745472,745472=401408} 2024-09-13T14:29:13.013 INFO:tasks.rados.rados.0.smithi026.stdout:2980: writing smithi02683976-35 from 0 to 745472 tid 1 2024-09-13T14:29:13.016 INFO:tasks.rados.rados.0.smithi026.stdout:2980: writing smithi02683976-35 from 745472 to 1146880 tid 2 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2981: copy_from oid 39 from oid 49 current snap is 288 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1414 (ObjNum 923 snap 287 seq_num 923) dirty exists 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2979: finishing copy_from to smithi02683976-15 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2979: got expected ENOENT (src dne) 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2979: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2977: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 0 (ObjNum 945 snap 0 seq_num 3913787808) dirty dne 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2977: done (4 left) 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2978: done (3 left) 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2979: done (2 left) 2024-09-13T14:29:13.017 INFO:tasks.rados.rados.0.smithi026.stdout:2982: delete oid 47 current snap is 288 2024-09-13T14:29:13.026 INFO:tasks.rados.rados.0.smithi026.stdout:2982: done (2 left) 2024-09-13T14:29:13.026 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 288 2024-09-13T14:29:13.026 INFO:tasks.rados.rados.0.smithi026.stdout:2983: seq_num 930 ranges {1441792=786432,2228224=401408} 2024-09-13T14:29:13.039 INFO:tasks.rados.rados.0.smithi026.stdout:2983: writing smithi02683976-46 from 1441792 to 2228224 tid 1 2024-09-13T14:29:13.042 INFO:tasks.rados.rados.0.smithi026.stdout:2983: writing smithi02683976-46 from 2228224 to 2629632 tid 2 2024-09-13T14:29:13.042 INFO:tasks.rados.rados.0.smithi026.stdout:2984: read oid 28 snap -1 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2984: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2985: rollback oid 8 current snap is 288 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 279 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2980: finishing write tid 1 to smithi02683976-35 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2986: rmattr oid 27 current snap is 288 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2986: done (5 left) 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2987: snap_create 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2980: finishing write tid 2 to smithi02683976-35 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2980: finishing write tid 3 to smithi02683976-35 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2981: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:29:13.043 INFO:tasks.rados.rados.0.smithi026.stdout:2981: finishing copy_from to smithi02683976-39 2024-09-13T14:29:13.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1192 (ObjNum 922 snap 287 seq_num 922) dirty exists 2024-09-13T14:29:13.044 INFO:tasks.rados.rados.0.smithi026.stdout:2980: finishing write tid 4 to smithi02683976-35 2024-09-13T14:29:13.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1748 (ObjNum 928 snap 288 seq_num 928) dirty exists 2024-09-13T14:29:13.044 INFO:tasks.rados.rados.0.smithi026.stdout:2980: left oid 35 (ObjNum 928 snap 288 seq_num 928) 2024-09-13T14:29:13.044 INFO:tasks.rados.rados.0.smithi026.stdout:2984: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:29:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:2983: finishing write tid 1 to smithi02683976-46 2024-09-13T14:29:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:2983: finishing write tid 2 to smithi02683976-46 2024-09-13T14:29:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:2983: finishing write tid 3 to smithi02683976-46 2024-09-13T14:29:13.047 INFO:tasks.rados.rados.0.smithi026.stdout:2985: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:29:13.047 INFO:tasks.rados.rados.0.smithi026.stdout:2983: finishing write tid 4 to smithi02683976-46 2024-09-13T14:29:13.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1417 (ObjNum 930 snap 288 seq_num 930) dirty exists 2024-09-13T14:29:13.047 INFO:tasks.rados.rados.0.smithi026.stdout:2983: left oid 46 (ObjNum 930 snap 288 seq_num 930) 2024-09-13T14:29:13.047 INFO:tasks.rados.rados.0.smithi026.stdout:2985: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:29:13.048 INFO:tasks.rados.rados.0.smithi026.stdout:2985: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:29:13.048 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1288 (ObjNum 860 snap 266 seq_num 860) dirty exists 2024-09-13T14:29:13.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:12 smithi026 ceph-mon[99461]: osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:29:13.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:12 smithi026 ceph-mon[97022]: osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:29:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:12 smithi145 ceph-mon[89145]: osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2980: done (5 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2981: done (4 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2983: done (3 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2984: done (2 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2985: done (1 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2987: done (0 left) 2024-09-13T14:29:13.990 INFO:tasks.rados.rados.0.smithi026.stdout:2988: snap_create 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[97022]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.4 MiB/s wr, 13 op/s 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[97022]: osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[97022]: osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[99461]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.4 MiB/s wr, 13 op/s 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[99461]: osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:29:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:13 smithi026 ceph-mon[99461]: osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:29:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:13 smithi145 ceph-mon[89145]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 5.4 MiB/s wr, 13 op/s 2024-09-13T14:29:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:13 smithi145 ceph-mon[89145]: osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:29:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:13 smithi145 ceph-mon[89145]: osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:29:14.991 INFO:tasks.rados.rados.0.smithi026.stdout:2988: done (0 left) 2024-09-13T14:29:14.992 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 290 2024-09-13T14:29:14.992 INFO:tasks.rados.rados.0.smithi026.stdout:2989: seq_num 931 ranges {2719744=802816,3522560=401408} 2024-09-13T14:29:15.011 INFO:tasks.rados.rados.0.smithi026.stdout:2989: writing smithi02683976-15 from 2719744 to 3522560 tid 1 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:2989: writing smithi02683976-15 from 3522560 to 3923968 tid 2 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:2990: rollback oid 3 current snap is 290 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 276 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:2991: read oid 50 snap 269 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:2991: expect deleted 2024-09-13T14:29:15.013 INFO:tasks.rados.rados.0.smithi026.stdout:2992: snap_create 2024-09-13T14:29:15.020 INFO:tasks.rados.rados.0.smithi026.stdout:2990: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:29:15.021 INFO:tasks.rados.rados.0.smithi026.stdout:2990: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:29:15.022 INFO:tasks.rados.rados.0.smithi026.stdout:2990: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:29:15.022 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1754 (ObjNum 879 snap 274 seq_num 879) dirty exists 2024-09-13T14:29:15.023 INFO:tasks.rados.rados.0.smithi026.stdout:2989: finishing write tid 1 to smithi02683976-15 2024-09-13T14:29:15.023 INFO:tasks.rados.rados.0.smithi026.stdout:2989: finishing write tid 2 to smithi02683976-15 2024-09-13T14:29:15.023 INFO:tasks.rados.rados.0.smithi026.stdout:2989: finishing write tid 3 to smithi02683976-15 2024-09-13T14:29:15.026 INFO:tasks.rados.rados.0.smithi026.stdout:2989: finishing write tid 4 to smithi02683976-15 2024-09-13T14:29:15.026 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1304 (ObjNum 931 snap 290 seq_num 931) dirty exists 2024-09-13T14:29:15.026 INFO:tasks.rados.rados.0.smithi026.stdout:2989: left oid 15 (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:16.007 INFO:tasks.rados.rados.0.smithi026.stdout:2989: done (3 left) 2024-09-13T14:29:16.007 INFO:tasks.rados.rados.0.smithi026.stdout:2990: done (2 left) 2024-09-13T14:29:16.007 INFO:tasks.rados.rados.0.smithi026.stdout:2991: done (1 left) 2024-09-13T14:29:16.007 INFO:tasks.rados.rados.0.smithi026.stdout:2992: done (0 left) 2024-09-13T14:29:16.007 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 291 2024-09-13T14:29:16.008 INFO:tasks.rados.rados.0.smithi026.stdout:2993: seq_num 932 ranges {0=417792,417792=663552,1081344=139264} 2024-09-13T14:29:16.009 INFO:tasks.rados.rados.0.smithi026.stdout:2993: writing smithi02683976-7 from 0 to 417792 tid 1 2024-09-13T14:29:16.013 INFO:tasks.rados.rados.0.smithi026.stdout:2993: writing smithi02683976-7 from 417792 to 1081344 tid 2 2024-09-13T14:29:16.014 INFO:tasks.rados.rados.0.smithi026.stdout:2993: writing smithi02683976-7 from 1081344 to 1220608 tid 3 2024-09-13T14:29:16.014 INFO:tasks.rados.rados.0.smithi026.stdout:2994: rollback oid 44 current snap is 291 2024-09-13T14:29:16.014 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 283 2024-09-13T14:29:16.014 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 291 2024-09-13T14:29:16.015 INFO:tasks.rados.rados.0.smithi026.stdout:2995: seq_num 933 ranges {0=434176,434176=548864,983040=253952} 2024-09-13T14:29:16.017 INFO:tasks.rados.rados.0.smithi026.stdout:2995: writing smithi02683976-42 from 0 to 434176 tid 1 2024-09-13T14:29:16.021 INFO:tasks.rados.rados.0.smithi026.stdout:2995: writing smithi02683976-42 from 434176 to 983040 tid 2 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2995: writing smithi02683976-42 from 983040 to 1236992 tid 3 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2996: rollback oid 9 current snap is 291 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2993: finishing write tid 1 to smithi02683976-7 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2993: finishing write tid 2 to smithi02683976-7 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2993: finishing write tid 3 to smithi02683976-7 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2993: finishing write tid 4 to smithi02683976-7 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2994: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2994: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1419 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 283 2024-09-13T14:29:16.024 INFO:tasks.rados.rados.0.smithi026.stdout:2994: done (3 left) 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2997: read oid 28 snap -1 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2997: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2993: finishing write tid 5 to smithi02683976-7 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1139 (ObjNum 932 snap 291 seq_num 932) dirty exists 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2993: left oid 7 (ObjNum 932 snap 291 seq_num 932) 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2998: read oid 2 snap -1 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2998: expect (ObjNum 913 snap 286 seq_num 913) 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2993: done (4 left) 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2999: rollback oid 27 current snap is 291 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 290 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:3000: setattr oid 22 current snap is 291 2024-09-13T14:29:16.025 INFO:tasks.rados.rados.0.smithi026.stdout:2995: finishing write tid 1 to smithi02683976-42 2024-09-13T14:29:16.026 INFO:tasks.rados.rados.0.smithi026.stdout:2997: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:2995: finishing write tid 2 to smithi02683976-42 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:2997: done (5 left) 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:3001: copy_from oid 37 from oid 11 current snap is 291 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:2995: finishing write tid 3 to smithi02683976-42 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:3002: rollback oid 38 current snap is 291 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 235 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:3003: read oid 4 snap -1 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:3003: expect (ObjNum 924 snap 287 seq_num 924) 2024-09-13T14:29:16.029 INFO:tasks.rados.rados.0.smithi026.stdout:3004: snap_create 2024-09-13T14:29:16.030 INFO:tasks.rados.rados.0.smithi026.stdout:2995: finishing write tid 4 to smithi02683976-42 2024-09-13T14:29:16.030 INFO:tasks.rados.rados.0.smithi026.stdout:2999: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:29:16.031 INFO:tasks.rados.rados.0.smithi026.stdout:2996: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:29:16.031 INFO:tasks.rados.rados.0.smithi026.stdout:2999: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:29:16.032 INFO:tasks.rados.rados.0.smithi026.stdout:2999: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:29:16.032 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1758 (ObjNum 915 snap 286 seq_num 915) dirty exists 2024-09-13T14:29:16.032 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1198 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:29:16.033 INFO:tasks.rados.rados.0.smithi026.stdout:2995: finishing write tid 5 to smithi02683976-42 2024-09-13T14:29:16.033 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1284 (ObjNum 933 snap 291 seq_num 933) dirty exists 2024-09-13T14:29:16.033 INFO:tasks.rados.rados.0.smithi026.stdout:2995: left oid 42 (ObjNum 933 snap 291 seq_num 933) 2024-09-13T14:29:16.034 INFO:tasks.rados.rados.0.smithi026.stdout:2998: expect (ObjNum 913 snap 286 seq_num 913) 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:2996: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:2996: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1488 (ObjNum 836 snap 261 seq_num 836) dirty exists 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:3002: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:3002: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:3002: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:29:16.062 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1420 (ObjNum 673 snap 189 seq_num 673) dirty exists 2024-09-13T14:29:16.063 INFO:tasks.rados.rados.0.smithi026.stdout:3001: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:29:16.063 INFO:tasks.rados.rados.0.smithi026.stdout:3001: finishing copy_from to smithi02683976-37 2024-09-13T14:29:16.063 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1455 (ObjNum 916 snap 286 seq_num 916) dirty exists 2024-09-13T14:29:16.063 INFO:tasks.rados.rados.0.smithi026.stdout:3003: expect (ObjNum 924 snap 287 seq_num 924) 2024-09-13T14:29:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:15 smithi026 ceph-mon[99461]: pgmap v578: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:15 smithi026 ceph-mon[99461]: osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:29:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:15 smithi026 ceph-mon[97022]: pgmap v578: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:16.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:15 smithi026 ceph-mon[97022]: osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:29:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:15 smithi145 ceph-mon[89145]: pgmap v578: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 460 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:29:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:15 smithi145 ceph-mon[89145]: osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:2995: done (8 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:2996: done (7 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:2998: done (6 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:2999: done (5 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:3000: done (4 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:3001: done (3 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:3002: done (2 left) 2024-09-13T14:29:17.014 INFO:tasks.rados.rados.0.smithi026.stdout:3003: done (1 left) 2024-09-13T14:29:17.015 INFO:tasks.rados.rados.0.smithi026.stdout:3004: done (0 left) 2024-09-13T14:29:17.015 INFO:tasks.rados.rados.0.smithi026.stdout:3005: delete oid 25 current snap is 292 2024-09-13T14:29:17.017 INFO:tasks.rados.rados.0.smithi026.stdout:3005: done (0 left) 2024-09-13T14:29:17.018 INFO:tasks.rados.rados.0.smithi026.stdout:3006: read oid 49 snap -1 2024-09-13T14:29:17.018 INFO:tasks.rados.rados.0.smithi026.stdout:3006: expect (ObjNum 922 snap 287 seq_num 922) 2024-09-13T14:29:17.018 INFO:tasks.rados.rados.0.smithi026.stdout:3007: delete oid 29 current snap is 292 2024-09-13T14:29:17.021 INFO:tasks.rados.rados.0.smithi026.stdout:3007: done (1 left) 2024-09-13T14:29:17.021 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 292 2024-09-13T14:29:17.021 INFO:tasks.rados.rados.0.smithi026.stdout:3008: seq_num 936 ranges {0=483328,483328=655360,1138688=147456} 2024-09-13T14:29:17.024 INFO:tasks.rados.rados.0.smithi026.stdout:3008: writing smithi02683976-20 from 0 to 483328 tid 1 2024-09-13T14:29:17.028 INFO:tasks.rados.rados.0.smithi026.stdout:3008: writing smithi02683976-20 from 483328 to 1138688 tid 2 2024-09-13T14:29:17.029 INFO:tasks.rados.rados.0.smithi026.stdout:3008: writing smithi02683976-20 from 1138688 to 1286144 tid 3 2024-09-13T14:29:17.029 INFO:tasks.rados.rados.0.smithi026.stdout:3009: copy_from oid 19 from oid 39 current snap is 292 2024-09-13T14:29:17.029 INFO:tasks.rados.rados.0.smithi026.stdout:3006: expect (ObjNum 922 snap 287 seq_num 922) 2024-09-13T14:29:17.061 INFO:tasks.rados.rados.0.smithi026.stdout:3008: finishing write tid 1 to smithi02683976-20 2024-09-13T14:29:17.061 INFO:tasks.rados.rados.0.smithi026.stdout:3008: finishing write tid 2 to smithi02683976-20 2024-09-13T14:29:17.061 INFO:tasks.rados.rados.0.smithi026.stdout:3008: finishing write tid 3 to smithi02683976-20 2024-09-13T14:29:17.061 INFO:tasks.rados.rados.0.smithi026.stdout:3008: finishing write tid 4 to smithi02683976-20 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3008: finishing write tid 5 to smithi02683976-20 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1599 (ObjNum 936 snap 292 seq_num 936) dirty exists 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3008: left oid 20 (ObjNum 936 snap 292 seq_num 936) 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3009: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3009: finishing copy_from to smithi02683976-19 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1192 (ObjNum 922 snap 287 seq_num 922) dirty exists 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3006: done (2 left) 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3008: done (1 left) 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3009: done (0 left) 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3010: read oid 47 snap 291 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3010: expect deleted 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:3011: rollback oid 28 current snap is 292 2024-09-13T14:29:17.062 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 235 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3012: read oid 18 snap -1 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3012: expect (ObjNum 884 snap 275 seq_num 884) 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3013: rollback oid 34 current snap is 292 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 289 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3014: copy_from oid 39 from oid 43 current snap is 292 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3015: copy_from oid 7 from oid 20 current snap is 292 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3016: read oid 40 snap -1 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3016: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3017: read oid 31 snap -1 2024-09-13T14:29:17.063 INFO:tasks.rados.rados.0.smithi026.stdout:3017: expect deleted 2024-09-13T14:29:17.064 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 292 2024-09-13T14:29:17.064 INFO:tasks.rados.rados.0.smithi026.stdout:3018: seq_num 937 ranges {1048576=499712,1548288=532480,2080768=270336} 2024-09-13T14:29:17.069 INFO:tasks.rados.rados.0.smithi026.stdout:3018: writing smithi02683976-19 from 1048576 to 1548288 tid 1 2024-09-13T14:29:17.073 INFO:tasks.rados.rados.0.smithi026.stdout:3018: writing smithi02683976-19 from 1548288 to 2080768 tid 2 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3018: writing smithi02683976-19 from 2080768 to 2351104 tid 3 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3019: read oid 24 snap 291 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3019: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3020: snap_create 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3014: finishing copy_from to smithi02683976-39 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3014: got expected ENOENT (src dne) 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3014: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:29:17.076 INFO:tasks.rados.rados.0.smithi026.stdout:3011: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3011: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3013: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3013: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3013: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1196 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3011: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1195 (ObjNum 767 snap 231 seq_num 767) dirty exists 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3015: finishing copy_from racing read to smithi02683976-7 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:3015: finishing copy_from to smithi02683976-7 2024-09-13T14:29:17.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1599 (ObjNum 936 snap 292 seq_num 936) dirty exists 2024-09-13T14:29:17.078 INFO:tasks.rados.rados.0.smithi026.stdout:3018: finishing write tid 1 to smithi02683976-19 2024-09-13T14:29:17.081 INFO:tasks.rados.rados.0.smithi026.stdout:3016: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:17.176 INFO:tasks.rados.rados.0.smithi026.stdout:3018: finishing write tid 2 to smithi02683976-19 2024-09-13T14:29:17.176 INFO:tasks.rados.rados.0.smithi026.stdout:3018: finishing write tid 3 to smithi02683976-19 2024-09-13T14:29:17.176 INFO:tasks.rados.rados.0.smithi026.stdout:3019: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:17.267 INFO:tasks.rados.rados.0.smithi026.stdout:3018: finishing write tid 4 to smithi02683976-19 2024-09-13T14:29:17.267 INFO:tasks.rados.rados.0.smithi026.stdout:3018: finishing write tid 5 to smithi02683976-19 2024-09-13T14:29:17.268 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1603 (ObjNum 937 snap 292 seq_num 937) dirty exists 2024-09-13T14:29:17.268 INFO:tasks.rados.rados.0.smithi026.stdout:3018: left oid 19 (ObjNum 937 snap 292 seq_num 937) 2024-09-13T14:29:17.268 INFO:tasks.rados.rados.0.smithi026.stdout:3012: expect (ObjNum 884 snap 275 seq_num 884) 2024-09-13T14:29:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:17 smithi026 ceph-mon[99461]: osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:29:17.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:17 smithi026 ceph-mon[97022]: osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:29:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:17 smithi145 ceph-mon[89145]: osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:29:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[97022]: pgmap v581: 121 pgs: 121 active+clean; 477 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.4 MiB/s wr, 11 op/s 2024-09-13T14:29:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[97022]: osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:29:18.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[97022]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[99461]: pgmap v581: 121 pgs: 121 active+clean; 477 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.4 MiB/s wr, 11 op/s 2024-09-13T14:29:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[99461]: osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:29:18.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:18 smithi026 ceph-mon[99461]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:18.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:29:18.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:18 smithi145 ceph-mon[89145]: pgmap v581: 121 pgs: 121 active+clean; 477 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.4 MiB/s wr, 11 op/s 2024-09-13T14:29:18.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:18 smithi145 ceph-mon[89145]: osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:29:18.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:18 smithi145 ceph-mon[89145]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:18.539 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3010: done (10 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3011: done (9 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3012: done (8 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3013: done (7 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3014: done (6 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3015: done (5 left) 2024-09-13T14:29:18.663 INFO:tasks.rados.rados.0.smithi026.stdout:3016: done (4 left) 2024-09-13T14:29:18.664 INFO:tasks.rados.rados.0.smithi026.stdout:3017: done (3 left) 2024-09-13T14:29:18.664 INFO:tasks.rados.rados.0.smithi026.stdout:3018: done (2 left) 2024-09-13T14:29:18.664 INFO:tasks.rados.rados.0.smithi026.stdout:3019: done (1 left) 2024-09-13T14:29:18.664 INFO:tasks.rados.rados.0.smithi026.stdout:3020: done (0 left) 2024-09-13T14:29:18.664 INFO:tasks.rados.rados.0.smithi026.stdout:3021: snap_remove snap 276 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3021: done (0 left) 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3022: read oid 1 snap -1 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3022: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3023: copy_from oid 12 from oid 10 current snap is 293 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3024: read oid 45 snap -1 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3024: expect deleted 2024-09-13T14:29:18.991 INFO:tasks.rados.rados.0.smithi026.stdout:3025: copy_from oid 28 from oid 16 current snap is 293 2024-09-13T14:29:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:3026: read oid 2 snap 291 2024-09-13T14:29:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:3026: expect (ObjNum 913 snap 286 seq_num 913) 2024-09-13T14:29:18.992 INFO:tasks.rados.rados.0.smithi026.stdout:3027: snap_create 2024-09-13T14:29:19.000 INFO:tasks.rados.rados.0.smithi026.stdout:3022: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:29:19.025 INFO:tasks.rados.rados.0.smithi026.stdout:3026: expect (ObjNum 913 snap 286 seq_num 913) 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:3025: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:3025: finishing copy_from to smithi02683976-28 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1152 (ObjNum 906 snap 282 seq_num 906) dirty exists 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:3023: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:3023: finishing copy_from to smithi02683976-12 2024-09-13T14:29:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1188 (ObjNum 835 snap 260 seq_num 835) dirty exists 2024-09-13T14:29:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:19 smithi026 ceph-mon[97022]: osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:29:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:19 smithi026 ceph-mon[97022]: osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:29:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:19 smithi026 ceph-mon[99461]: osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:29:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:19 smithi026 ceph-mon[99461]: osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:29:19.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:19 smithi145 ceph-mon[89145]: osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:29:19.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:19 smithi145 ceph-mon[89145]: osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:29:19.992 INFO:tasks.rados.rados.0.smithi026.stdout:3022: done (5 left) 2024-09-13T14:29:19.993 INFO:tasks.rados.rados.0.smithi026.stdout:3023: done (4 left) 2024-09-13T14:29:19.993 INFO:tasks.rados.rados.0.smithi026.stdout:3024: done (3 left) 2024-09-13T14:29:19.993 INFO:tasks.rados.rados.0.smithi026.stdout:3025: done (2 left) 2024-09-13T14:29:19.993 INFO:tasks.rados.rados.0.smithi026.stdout:3026: done (1 left) 2024-09-13T14:29:19.993 INFO:tasks.rados.rados.0.smithi026.stdout:3027: done (0 left) 2024-09-13T14:29:19.994 INFO:tasks.rados.rados.0.smithi026.stdout:3028: setattr oid 18 current snap is 294 2024-09-13T14:29:19.995 INFO:tasks.rados.rados.0.smithi026.stdout:3029: copy_from oid 28 from oid 41 current snap is 294 2024-09-13T14:29:19.995 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 294 2024-09-13T14:29:19.995 INFO:tasks.rados.rados.0.smithi026.stdout:3030: seq_num 938 ranges {9904128=516096,10420224=417792,10838016=385024} 2024-09-13T14:29:20.052 INFO:tasks.rados.rados.0.smithi026.stdout:3030: writing smithi02683976-3 from 9904128 to 10420224 tid 1 2024-09-13T14:29:20.055 INFO:tasks.rados.rados.0.smithi026.stdout:3030: writing smithi02683976-3 from 10420224 to 10838016 tid 2 2024-09-13T14:29:20.058 INFO:tasks.rados.rados.0.smithi026.stdout:3030: writing smithi02683976-3 from 10838016 to 11223040 tid 3 2024-09-13T14:29:20.058 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 294 2024-09-13T14:29:20.058 INFO:tasks.rados.rados.0.smithi026.stdout:3031: seq_num 939 ranges {1556480=532480,2088960=704512,2793472=98304} 2024-09-13T14:29:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:3031: writing smithi02683976-5 from 1556480 to 2088960 tid 1 2024-09-13T14:29:20.074 INFO:tasks.rados.rados.0.smithi026.stdout:3031: writing smithi02683976-5 from 2088960 to 2793472 tid 2 2024-09-13T14:29:20.074 INFO:tasks.rados.rados.0.smithi026.stdout:3031: writing smithi02683976-5 from 2793472 to 2891776 tid 3 2024-09-13T14:29:20.075 INFO:tasks.rados.rados.0.smithi026.stdout:3032: snap_remove snap 247 2024-09-13T14:29:20.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:20 smithi026 ceph-mon[97022]: pgmap v584: 121 pgs: 121 active+clean; 479 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:20 smithi026 ceph-mon[97022]: osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:29:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:20 smithi026 ceph-mon[99461]: pgmap v584: 121 pgs: 121 active+clean; 479 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:20 smithi026 ceph-mon[99461]: osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:29:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:20 smithi145 ceph-mon[89145]: pgmap v584: 121 pgs: 121 active+clean; 479 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:20 smithi145 ceph-mon[89145]: osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:29:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (8m) 70s ago 12m 18.4M - 0.27.0 11f11916f8cd 4b02ef02803f 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi145 *:3000 running (31s) 25s ago 12m 45.8M - c8b91775d855 4b007f928313 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mds.a smithi026 running (74s) 70s ago 14m 14.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 e163e8cfc713 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mds.b smithi145 running (53s) 25s ago 14m 13.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 859c720bf426 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi145 *:8443,9283,8765 running (7m) 25s ago 17m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 6f4dd502efd7 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (7m) 70s ago 19m 529M - 19.3.0-4697-gd58f7306 fec07016a2b0 1443c0c3060c 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 70s ago 19m 66.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 fde66f55593e 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi145 running (6m) 25s ago 18m 55.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 22caa9373f03 2024-09-13T14:29:20.476 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 70s ago 17m 55.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 f85c39c40108 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 70s ago 12m 11.1M - 1.7.0 72c9c2088986 21fa336e551b 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi145 *:9100 running (7m) 25s ago 12m 12.4M - 1.7.0 72c9c2088986 a22f428718b1 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 70s ago 17m 242M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3429eb79404f 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 70s ago 16m 240M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2441dc634f7c 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 70s ago 16m 193M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 581bbbb517ea 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 70s ago 16m 96.1M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 2146eb0c6926 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi145 running (3m) 25s ago 15m 186M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 1d10fdf8c82a 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi145 running (2m) 25s ago 15m 163M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 b9422108344c 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi145 running (2m) 25s ago 14m 110M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 304ffd9a14d8 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi145 running (92s) 25s ago 14m 87.6M 2480M 19.3.0-4697-gd58f7306 fec07016a2b0 606a2c6e152f 2024-09-13T14:29:20.477 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi145 *:9095 running (7m) 25s ago 13m 54.1M - 2.51.0 1d3b7f56885b 106373b0ae43 2024-09-13T14:29:21.034 INFO:tasks.rados.rados.0.smithi026.stdout:3032: done (4 left) 2024-09-13T14:29:21.034 INFO:tasks.rados.rados.0.smithi026.stdout:3033: rollback oid 20 current snap is 294 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 293 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:3034: setattr oid 39 current snap is 294 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1430 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:3029: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:3029: finishing copy_from to smithi02683976-28 2024-09-13T14:29:21.035 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 944 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:29:21.036 INFO:tasks.rados.rados.0.smithi026.stdout:3030: finishing write tid 1 to smithi02683976-3 2024-09-13T14:29:21.036 INFO:tasks.rados.rados.0.smithi026.stdout:3030: finishing write tid 2 to smithi02683976-3 2024-09-13T14:29:21.036 INFO:tasks.rados.rados.0.smithi026.stdout:3030: finishing write tid 3 to smithi02683976-3 2024-09-13T14:29:21.036 INFO:tasks.rados.rados.0.smithi026.stdout:3030: finishing write tid 4 to smithi02683976-3 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3030: finishing write tid 5 to smithi02683976-3 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1769 (ObjNum 938 snap 294 seq_num 938) dirty exists 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3030: left oid 3 (ObjNum 938 snap 294 seq_num 938) 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: finishing write tid 1 to smithi02683976-5 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: finishing write tid 2 to smithi02683976-5 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: finishing write tid 3 to smithi02683976-5 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: finishing write tid 4 to smithi02683976-5 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: finishing write tid 5 to smithi02683976-5 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1774 (ObjNum 939 snap 294 seq_num 939) dirty exists 2024-09-13T14:29:21.037 INFO:tasks.rados.rados.0.smithi026.stdout:3031: left oid 5 (ObjNum 939 snap 294 seq_num 939) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3028: done (5 left) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3029: done (4 left) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3030: done (3 left) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3031: done (2 left) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3035: copy_from oid 45 from oid 34 current snap is 294 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3036: read oid 15 snap -1 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3036: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:21.038 INFO:tasks.rados.rados.0.smithi026.stdout:3037: snap_remove snap 264 2024-09-13T14:29:21.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions' 2024-09-13T14:29:21.477 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:21.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:21] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.51.0" 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:3037: done (4 left) 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:3038: rollback oid 49 current snap is 294 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 49 to 285 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1272 (ObjNum 922 snap 287 seq_num 922) dirty exists 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:3033: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:3034: done (4 left) 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 294 2024-09-13T14:29:22.038 INFO:tasks.rados.rados.0.smithi026.stdout:3039: seq_num 940 ranges {3473408=548864,4022272=589824,4612096=212992} 2024-09-13T14:29:22.062 INFO:tasks.rados.rados.0.smithi026.stdout:3039: writing smithi02683976-10 from 3473408 to 4022272 tid 1 2024-09-13T14:29:22.066 INFO:tasks.rados.rados.0.smithi026.stdout:3039: writing smithi02683976-10 from 4022272 to 4612096 tid 2 2024-09-13T14:29:22.067 INFO:tasks.rados.rados.0.smithi026.stdout:3039: writing smithi02683976-10 from 4612096 to 4825088 tid 3 2024-09-13T14:29:22.067 INFO:tasks.rados.rados.0.smithi026.stdout:3040: snap_remove snap 292 2024-09-13T14:29:22.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[97022]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:22.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[97022]: pgmap v587: 121 pgs: 121 active+clean; 504 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-13T14:29:22.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[97022]: osdmap e657: 8 total, 8 up, 8 in 2024-09-13T14:29:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[99461]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[99461]: pgmap v587: 121 pgs: 121 active+clean; 504 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-13T14:29:22.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:22 smithi026 ceph-mon[99461]: osdmap e657: 8 total, 8 up, 8 in 2024-09-13T14:29:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:22 smithi145 ceph-mon[89145]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:22 smithi145 ceph-mon[89145]: pgmap v587: 121 pgs: 121 active+clean; 504 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-09-13T14:29:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:22 smithi145 ceph-mon[89145]: osdmap e657: 8 total, 8 up, 8 in 2024-09-13T14:29:23.045 INFO:tasks.rados.rados.0.smithi026.stdout:3040: done (5 left) 2024-09-13T14:29:23.045 INFO:tasks.rados.rados.0.smithi026.stdout:3041: delete oid 4 current snap is 294 2024-09-13T14:29:23.046 INFO:tasks.rados.rados.0.smithi026.stdout:3033: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:29:23.047 INFO:tasks.rados.rados.0.smithi026.stdout:3033: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:29:23.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1606 (ObjNum 936 snap 292 seq_num 936) dirty exists 2024-09-13T14:29:23.047 INFO:tasks.rados.rados.0.smithi026.stdout:3036: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3035: finishing copy_from racing read to smithi02683976-45 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3035: finishing copy_from to smithi02683976-45 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1196 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3033: done (5 left) 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3035: done (4 left) 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3036: done (3 left) 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3041: done (2 left) 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3042: delete oid 17 current snap is 294 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3038: finishing rollback tid 0 to smithi02683976-49 2024-09-13T14:29:23.219 INFO:tasks.rados.rados.0.smithi026.stdout:3038: finishing rollback tid 1 to smithi02683976-49 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1197 (ObjNum 65 snap 0 seq_num 1600089941) dirty dne 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: finishing write tid 1 to smithi02683976-10 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: finishing write tid 2 to smithi02683976-10 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: finishing write tid 3 to smithi02683976-10 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: finishing write tid 4 to smithi02683976-10 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: finishing write tid 5 to smithi02683976-10 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1201 (ObjNum 940 snap 294 seq_num 940) dirty exists 2024-09-13T14:29:23.220 INFO:tasks.rados.rados.0.smithi026.stdout:3039: left oid 10 (ObjNum 940 snap 294 seq_num 940) 2024-09-13T14:29:23.221 INFO:tasks.rados.rados.0.smithi026.stdout:3038: done (2 left) 2024-09-13T14:29:23.222 INFO:tasks.rados.rados.0.smithi026.stdout:3039: done (1 left) 2024-09-13T14:29:23.222 INFO:tasks.rados.rados.0.smithi026.stdout:3042: done (0 left) 2024-09-13T14:29:23.222 INFO:tasks.rados.rados.0.smithi026.stdout:3043: rmattr oid 32 current snap is 294 2024-09-13T14:29:23.222 INFO:tasks.rados.rados.0.smithi026.stdout:3044: snap_remove snap 269 2024-09-13T14:29:23.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:23 smithi026 ceph-mon[99461]: osdmap e658: 8 total, 8 up, 8 in 2024-09-13T14:29:23.653 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:23 smithi026 ceph-mon[97022]: osdmap e658: 8 total, 8 up, 8 in 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 8 2024-09-13T14:29:23.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 15 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-09-13T14:29:23.655 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-09-13T14:29:23.655 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:23 smithi145 ceph-mon[89145]: osdmap e658: 8 total, 8 up, 8 in 2024-09-13T14:29:23.995 INFO:tasks.rados.rados.0.smithi026.stdout:3044: done (1 left) 2024-09-13T14:29:23.995 INFO:tasks.rados.rados.0.smithi026.stdout:3045: read oid 29 snap -1 2024-09-13T14:29:23.996 INFO:tasks.rados.rados.0.smithi026.stdout:3045: expect deleted 2024-09-13T14:29:23.996 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 294 2024-09-13T14:29:23.996 INFO:tasks.rados.rados.0.smithi026.stdout:3046: seq_num 943 ranges {0=598016,598016=688128,1286144=114688} 2024-09-13T14:29:23.996 INFO:tasks.rados.rados.0.smithi026.stdout:3046: writing smithi02683976-6 from 0 to 598016 tid 1 2024-09-13T14:29:24.001 INFO:tasks.rados.rados.0.smithi026.stdout:3046: writing smithi02683976-6 from 598016 to 1286144 tid 2 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:3046: writing smithi02683976-6 from 1286144 to 1400832 tid 3 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1448 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:3043: done (2 left) 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:3045: done (1 left) 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 294 2024-09-13T14:29:24.003 INFO:tasks.rados.rados.0.smithi026.stdout:3047: seq_num 944 ranges {4825088=614400,5439488=573440,6012928=229376} 2024-09-13T14:29:24.034 INFO:tasks.rados.rados.0.smithi026.stdout:3047: writing smithi02683976-10 from 4825088 to 5439488 tid 1 2024-09-13T14:29:24.037 INFO:tasks.rados.rados.0.smithi026.stdout:3047: writing smithi02683976-10 from 5439488 to 6012928 tid 2 2024-09-13T14:29:24.039 INFO:tasks.rados.rados.0.smithi026.stdout:3047: writing smithi02683976-10 from 6012928 to 6242304 tid 3 2024-09-13T14:29:24.039 INFO:tasks.rados.rados.0.smithi026.stdout:3048: read oid 14 snap -1 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3048: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3049: setattr oid 1 current snap is 294 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: finishing write tid 1 to smithi02683976-6 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: finishing write tid 2 to smithi02683976-6 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: finishing write tid 3 to smithi02683976-6 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: finishing write tid 4 to smithi02683976-6 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: finishing write tid 5 to smithi02683976-6 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1516 (ObjNum 943 snap 294 seq_num 943) dirty exists 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3046: left oid 6 (ObjNum 943 snap 294 seq_num 943) 2024-09-13T14:29:24.040 INFO:tasks.rados.rados.0.smithi026.stdout:3047: finishing write tid 1 to smithi02683976-10 2024-09-13T14:29:24.041 INFO:tasks.rados.rados.0.smithi026.stdout:3046: done (3 left) 2024-09-13T14:29:24.041 INFO:tasks.rados.rados.0.smithi026.stdout:3050: snap_create 2024-09-13T14:29:24.045 INFO:tasks.rados.rados.0.smithi026.stdout:3047: finishing write tid 2 to smithi02683976-10 2024-09-13T14:29:24.045 INFO:tasks.rados.rados.0.smithi026.stdout:3047: finishing write tid 3 to smithi02683976-10 2024-09-13T14:29:24.045 INFO:tasks.rados.rados.0.smithi026.stdout:3047: finishing write tid 4 to smithi02683976-10 2024-09-13T14:29:24.046 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1806 (ObjNum 893 snap 277 seq_num 893) dirty exists 2024-09-13T14:29:24.047 INFO:tasks.rados.rados.0.smithi026.stdout:3047: finishing write tid 5 to smithi02683976-10 2024-09-13T14:29:24.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1205 (ObjNum 944 snap 294 seq_num 944) dirty exists 2024-09-13T14:29:24.047 INFO:tasks.rados.rados.0.smithi026.stdout:3047: left oid 10 (ObjNum 944 snap 294 seq_num 944) 2024-09-13T14:29:24.088 INFO:tasks.rados.rados.0.smithi026.stdout:3048: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-13T14:29:24.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[97022]: pgmap v590: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 32 op/s 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[97022]: osdmap e659: 8 total, 8 up, 8 in 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1776054100' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[97022]: osdmap e660: 8 total, 8 up, 8 in 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[99461]: pgmap v590: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 32 op/s 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[99461]: osdmap e659: 8 total, 8 up, 8 in 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1776054100' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:24.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:24 smithi026 ceph-mon[99461]: osdmap e660: 8 total, 8 up, 8 in 2024-09-13T14:29:24.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T14:29:24.443 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:24 smithi145 ceph-mon[89145]: pgmap v590: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 32 op/s 2024-09-13T14:29:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:24 smithi145 ceph-mon[89145]: osdmap e659: 8 total, 8 up, 8 in 2024-09-13T14:29:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:24 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1776054100' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:24 smithi145 ceph-mon[89145]: osdmap e660: 8 total, 8 up, 8 in 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:3047: done (3 left) 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:3048: done (2 left) 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:3049: done (1 left) 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:3050: done (0 left) 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 295 2024-09-13T14:29:25.713 INFO:tasks.rados.rados.0.smithi026.stdout:3051: seq_num 945 ranges {12591104=630784,13221888=458752,13680640=344064} 2024-09-13T14:29:25.785 INFO:tasks.rados.rados.0.smithi026.stdout:3051: writing smithi02683976-14 from 12591104 to 13221888 tid 1 2024-09-13T14:29:25.787 INFO:tasks.rados.rados.0.smithi026.stdout:3051: writing smithi02683976-14 from 13221888 to 13680640 tid 2 2024-09-13T14:29:25.790 INFO:tasks.rados.rados.0.smithi026.stdout:3051: writing smithi02683976-14 from 13680640 to 14024704 tid 3 2024-09-13T14:29:25.790 INFO:tasks.rados.rados.0.smithi026.stdout:3052: rollback oid 15 current snap is 295 2024-09-13T14:29:25.790 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 15 to 291 2024-09-13T14:29:25.790 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 295 2024-09-13T14:29:25.790 INFO:tasks.rados.rados.0.smithi026.stdout:3053: seq_num 946 ranges {2629632=647168,3276800=745472,4022272=57344} 2024-09-13T14:29:25.808 INFO:tasks.rados.rados.0.smithi026.stdout:3053: writing smithi02683976-46 from 2629632 to 3276800 tid 1 2024-09-13T14:29:25.815 INFO:tasks.rados.rados.0.smithi026.stdout:3053: writing smithi02683976-46 from 3276800 to 4022272 tid 2 2024-09-13T14:29:25.815 INFO:tasks.rados.rados.0.smithi026.stdout:3053: writing smithi02683976-46 from 4022272 to 4079616 tid 3 2024-09-13T14:29:25.815 INFO:tasks.rados.rados.0.smithi026.stdout:3054: read oid 49 snap 290 2024-09-13T14:29:25.815 INFO:tasks.rados.rados.0.smithi026.stdout:3054: expect (ObjNum 922 snap 287 seq_num 922) 2024-09-13T14:29:25.815 INFO:tasks.rados.rados.0.smithi026.stdout:3055: rmattr oid 8 current snap is 295 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:3051: finishing write tid 1 to smithi02683976-14 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:3051: finishing write tid 2 to smithi02683976-14 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:3051: finishing write tid 3 to smithi02683976-14 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:3051: finishing write tid 4 to smithi02683976-14 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:3051: finishing write tid 5 to smithi02683976-14 2024-09-13T14:29:25.816 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1461 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:3051: left oid 14 (ObjNum 945 snap 295 seq_num 945) 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:3052: finishing rollback tid 0 to smithi02683976-15 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:3052: finishing rollback tid 1 to smithi02683976-15 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:3052: finishing rollback tid 2 to smithi02683976-15 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1308 (ObjNum 931 snap 290 seq_num 931) dirty exists 2024-09-13T14:29:25.817 INFO:tasks.rados.rados.0.smithi026.stdout:3051: done (4 left) 2024-09-13T14:29:25.818 INFO:tasks.rados.rados.0.smithi026.stdout:3052: done (3 left) 2024-09-13T14:29:25.818 INFO:tasks.rados.rados.0.smithi026.stdout:3056: snap_create 2024-09-13T14:29:25.818 INFO:tasks.rados.rados.0.smithi026.stdout:3053: finishing write tid 1 to smithi02683976-46 2024-09-13T14:29:25.819 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1289 (ObjNum 860 snap 266 seq_num 860) dirty exists 2024-09-13T14:29:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:3053: finishing write tid 2 to smithi02683976-46 2024-09-13T14:29:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:3053: finishing write tid 3 to smithi02683976-46 2024-09-13T14:29:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:3053: finishing write tid 4 to smithi02683976-46 2024-09-13T14:29:25.820 INFO:tasks.rados.rados.0.smithi026.stdout:3054: expect (ObjNum 922 snap 287 seq_num 922) 2024-09-13T14:29:25.847 INFO:tasks.rados.rados.0.smithi026.stdout:3053: finishing write tid 5 to smithi02683976-46 2024-09-13T14:29:25.847 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1423 (ObjNum 946 snap 295 seq_num 946) dirty exists 2024-09-13T14:29:25.847 INFO:tasks.rados.rados.0.smithi026.stdout:3053: left oid 46 (ObjNum 946 snap 295 seq_num 946) 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3053: done (3 left) 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3054: done (2 left) 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3055: done (1 left) 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3056: done (0 left) 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3057: read oid 5 snap -1 2024-09-13T14:29:26.056 INFO:tasks.rados.rados.0.smithi026.stdout:3057: expect (ObjNum 939 snap 294 seq_num 939) 2024-09-13T14:29:26.057 INFO:tasks.rados.rados.0.smithi026.stdout:3058: read oid 42 snap 285 2024-09-13T14:29:26.057 INFO:tasks.rados.rados.0.smithi026.stdout:3058: expect deleted 2024-09-13T14:29:26.057 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 296 2024-09-13T14:29:26.057 INFO:tasks.rados.rados.0.smithi026.stdout:3059: seq_num 947 ranges {901120=663552,1564672=630784,2195456=172032} 2024-09-13T14:29:26.065 INFO:tasks.rados.rados.0.smithi026.stdout:3059: writing smithi02683976-2 from 901120 to 1564672 tid 1 2024-09-13T14:29:26.069 INFO:tasks.rados.rados.0.smithi026.stdout:3059: writing smithi02683976-2 from 1564672 to 2195456 tid 2 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3059: writing smithi02683976-2 from 2195456 to 2367488 tid 3 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3060: rmattr oid 47 current snap is 296 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3060: done (3 left) 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3061: read oid 9 snap -1 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3061: expect (ObjNum 836 snap 261 seq_num 836) 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 296 2024-09-13T14:29:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3062: seq_num 948 ranges {0=679936,679936=516096,1196032=286720} 2024-09-13T14:29:26.074 INFO:tasks.rados.rados.0.smithi026.stdout:3062: writing smithi02683976-13 from 0 to 679936 tid 1 2024-09-13T14:29:26.077 INFO:tasks.rados.rados.0.smithi026.stdout:3062: writing smithi02683976-13 from 679936 to 1196032 tid 2 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3062: writing smithi02683976-13 from 1196032 to 1482752 tid 3 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3063: rollback oid 20 current snap is 296 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 282 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3064: read oid 29 snap 291 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3064: expect (ObjNum 901 snap 278 seq_num 901) 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 296 2024-09-13T14:29:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3057: expect (ObjNum 939 snap 294 seq_num 939) 2024-09-13T14:29:26.204 INFO:tasks.rados.rados.0.smithi026.stdout:3059: finishing write tid 1 to smithi02683976-2 2024-09-13T14:29:26.204 INFO:tasks.rados.rados.0.smithi026.stdout:3059: finishing write tid 2 to smithi02683976-2 2024-09-13T14:29:26.205 INFO:tasks.rados.rados.0.smithi026.stdout:3059: finishing write tid 3 to smithi02683976-2 2024-09-13T14:29:26.205 INFO:tasks.rados.rados.0.smithi026.stdout:3059: finishing write tid 4 to smithi02683976-2 2024-09-13T14:29:26.205 INFO:tasks.rados.rados.0.smithi026.stdout:3065: seq_num 949 ranges {3121152=704512,3825664=794624,4620288=8192} 2024-09-13T14:29:26.220 INFO:tasks.rados.rados.0.smithi026.stdout:3065: writing smithi02683976-44 from 3121152 to 3825664 tid 1 2024-09-13T14:29:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:3065: writing smithi02683976-44 from 3825664 to 4620288 tid 2 2024-09-13T14:29:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:3065: writing smithi02683976-44 from 4620288 to 4628480 tid 3 2024-09-13T14:29:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:3057: done (7 left) 2024-09-13T14:29:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:3058: done (6 left) 2024-09-13T14:29:26.225 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 296 2024-09-13T14:29:26.226 INFO:tasks.rados.rados.0.smithi026.stdout:3059: finishing write tid 5 to smithi02683976-2 2024-09-13T14:29:26.226 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1276 (ObjNum 947 snap 296 seq_num 947) dirty exists 2024-09-13T14:29:26.226 INFO:tasks.rados.rados.0.smithi026.stdout:3059: left oid 2 (ObjNum 947 snap 296 seq_num 947) 2024-09-13T14:29:26.226 INFO:tasks.rados.rados.0.smithi026.stdout:3062: finishing write tid 1 to smithi02683976-13 2024-09-13T14:29:26.226 INFO:tasks.rados.rados.0.smithi026.stdout:3066: seq_num 950 ranges {0=720896,720896=679936,1400832=122880} 2024-09-13T14:29:26.229 INFO:tasks.rados.rados.0.smithi026.stdout:3066: writing smithi02683976-17 from 0 to 720896 tid 1 2024-09-13T14:29:26.233 INFO:tasks.rados.rados.0.smithi026.stdout:3066: writing smithi02683976-17 from 720896 to 1400832 tid 2 2024-09-13T14:29:26.234 INFO:tasks.rados.rados.0.smithi026.stdout:3066: writing smithi02683976-17 from 1400832 to 1523712 tid 3 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3059: done (6 left) 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3067: read oid 21 snap -1 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3067: expect (ObjNum 906 snap 282 seq_num 906) 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3068: copy_from oid 26 from oid 32 current snap is 296 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3062: finishing write tid 2 to smithi02683976-13 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3062: finishing write tid 3 to smithi02683976-13 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3062: finishing write tid 4 to smithi02683976-13 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3063: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3062: finishing write tid 5 to smithi02683976-13 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1610 (ObjNum 948 snap 296 seq_num 948) dirty exists 2024-09-13T14:29:26.235 INFO:tasks.rados.rados.0.smithi026.stdout:3062: left oid 13 (ObjNum 948 snap 296 seq_num 948) 2024-09-13T14:29:26.236 INFO:tasks.rados.rados.0.smithi026.stdout:3063: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:29:26.236 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1612 (ObjNum 33 snap 0 seq_num 5) dirty dne 2024-09-13T14:29:26.236 INFO:tasks.rados.rados.0.smithi026.stdout:3061: expect (ObjNum 836 snap 261 seq_num 836) 2024-09-13T14:29:26.272 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:26 smithi026 ceph-mon[99461]: pgmap v593: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-13T14:29:26.273 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:26 smithi026 ceph-mon[99461]: osdmap e661: 8 total, 8 up, 8 in 2024-09-13T14:29:26.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:26 smithi026 ceph-mon[97022]: pgmap v593: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-13T14:29:26.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:26 smithi026 ceph-mon[97022]: osdmap e661: 8 total, 8 up, 8 in 2024-09-13T14:29:26.370 INFO:tasks.rados.rados.0.smithi026.stdout:3064: expect (ObjNum 901 snap 278 seq_num 901) 2024-09-13T14:29:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:26 smithi145 ceph-mon[89145]: pgmap v593: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 502 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-13T14:29:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:26 smithi145 ceph-mon[89145]: osdmap e661: 8 total, 8 up, 8 in 2024-09-13T14:29:26.538 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: finishing write tid 1 to smithi02683976-44 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: finishing write tid 2 to smithi02683976-44 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: finishing write tid 3 to smithi02683976-44 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: finishing write tid 4 to smithi02683976-44 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: finishing write tid 5 to smithi02683976-44 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1427 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3065: left oid 44 (ObjNum 949 snap 296 seq_num 949) 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3066: finishing write tid 1 to smithi02683976-17 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3066: finishing write tid 2 to smithi02683976-17 2024-09-13T14:29:26.545 INFO:tasks.rados.rados.0.smithi026.stdout:3061: done (7 left) 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3062: done (6 left) 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3063: done (5 left) 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3064: done (4 left) 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3065: done (3 left) 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 296 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3066: finishing write tid 3 to smithi02683976-17 2024-09-13T14:29:26.546 INFO:tasks.rados.rados.0.smithi026.stdout:3069: seq_num 951 ranges {11223040=737280,11960320=614400,12574720=188416} 2024-09-13T14:29:26.610 INFO:tasks.rados.rados.0.smithi026.stdout:3069: writing smithi02683976-3 from 11223040 to 11960320 tid 1 2024-09-13T14:29:26.613 INFO:tasks.rados.rados.0.smithi026.stdout:3069: writing smithi02683976-3 from 11960320 to 12574720 tid 2 2024-09-13T14:29:26.614 INFO:tasks.rados.rados.0.smithi026.stdout:3069: writing smithi02683976-3 from 12574720 to 12763136 tid 3 2024-09-13T14:29:26.614 INFO:tasks.rados.rados.0.smithi026.stdout:3070: copy_from oid 38 from oid 20 current snap is 296 2024-09-13T14:29:26.615 INFO:tasks.rados.rados.0.smithi026.stdout:3066: finishing write tid 4 to smithi02683976-17 2024-09-13T14:29:26.615 INFO:tasks.rados.rados.0.smithi026.stdout:3066: finishing write tid 5 to smithi02683976-17 2024-09-13T14:29:26.615 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1201 (ObjNum 950 snap 296 seq_num 950) dirty exists 2024-09-13T14:29:26.615 INFO:tasks.rados.rados.0.smithi026.stdout:3066: left oid 17 (ObjNum 950 snap 296 seq_num 950) 2024-09-13T14:29:26.615 INFO:tasks.rados.rados.0.smithi026.stdout:3067: expect (ObjNum 906 snap 282 seq_num 906) 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:3068: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:3068: finishing copy_from to smithi02683976-26 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1448 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:3066: done (4 left) 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:3067: done (3 left) 2024-09-13T14:29:26.718 INFO:tasks.rados.rados.0.smithi026.stdout:3068: done (2 left) 2024-09-13T14:29:26.719 INFO:tasks.rados.rados.0.smithi026.stdout:3071: snap_remove snap 283 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3071: done (2 left) 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3072: rollback oid 27 current snap is 296 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 271 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3073: delete oid 48 current snap is 296 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3070: finishing copy_from to smithi02683976-38 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3070: got expected ENOENT (src dne) 2024-09-13T14:29:27.060 INFO:tasks.rados.rados.0.smithi026.stdout:3069: finishing write tid 1 to smithi02683976-3 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3070: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3069: finishing write tid 2 to smithi02683976-3 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3069: finishing write tid 3 to smithi02683976-3 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3069: finishing write tid 4 to smithi02683976-3 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3069: finishing write tid 5 to smithi02683976-3 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1811 (ObjNum 951 snap 296 seq_num 951) dirty exists 2024-09-13T14:29:27.061 INFO:tasks.rados.rados.0.smithi026.stdout:3069: left oid 3 (ObjNum 951 snap 296 seq_num 951) 2024-09-13T14:29:27.070 INFO:tasks.rados.rados.0.smithi026.stdout:3069: done (3 left) 2024-09-13T14:29:27.070 INFO:tasks.rados.rados.0.smithi026.stdout:3070: done (2 left) 2024-09-13T14:29:27.070 INFO:tasks.rados.rados.0.smithi026.stdout:3073: done (1 left) 2024-09-13T14:29:27.070 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 296 2024-09-13T14:29:27.070 INFO:tasks.rados.rados.0.smithi026.stdout:3074: seq_num 953 ranges {0=770048,770048=786432,1556480=16384} 2024-09-13T14:29:27.075 INFO:tasks.rados.rados.0.smithi026.stdout:3074: writing smithi02683976-4 from 0 to 770048 tid 1 2024-09-13T14:29:27.082 INFO:tasks.rados.rados.0.smithi026.stdout:3074: writing smithi02683976-4 from 770048 to 1556480 tid 2 2024-09-13T14:29:27.082 INFO:tasks.rados.rados.0.smithi026.stdout:3074: writing smithi02683976-4 from 1556480 to 1572864 tid 3 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3075: read oid 6 snap 290 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3075: expect deleted 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3076: copy_from oid 21 from oid 24 current snap is 296 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3072: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3072: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3072: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1815 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3074: finishing write tid 1 to smithi02683976-4 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3072: done (3 left) 2024-09-13T14:29:27.083 INFO:tasks.rados.rados.0.smithi026.stdout:3077: delete oid 10 current snap is 296 2024-09-13T14:29:27.087 INFO:tasks.rados.rados.0.smithi026.stdout:3074: finishing write tid 2 to smithi02683976-4 2024-09-13T14:29:27.087 INFO:tasks.rados.rados.0.smithi026.stdout:3074: finishing write tid 3 to smithi02683976-4 2024-09-13T14:29:27.087 INFO:tasks.rados.rados.0.smithi026.stdout:3075: done (3 left) 2024-09-13T14:29:27.087 INFO:tasks.rados.rados.0.smithi026.stdout:3077: done (2 left) 2024-09-13T14:29:27.087 INFO:tasks.rados.rados.0.smithi026.stdout:3078: snap_remove snap 282 2024-09-13T14:29:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[97022]: osdmap e662: 8 total, 8 up, 8 in 2024-09-13T14:29:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:27.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3621465959' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[99461]: osdmap e662: 8 total, 8 up, 8 in 2024-09-13T14:29:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:27.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:27 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3621465959' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:27.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad455616-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T14:29:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:27 smithi145 ceph-mon[89145]: osdmap e662: 8 total, 8 up, 8 in 2024-09-13T14:29:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:27 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3621465959' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:27.462 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:28.076 INFO:tasks.rados.rados.0.smithi026.stdout:3078: done (2 left) 2024-09-13T14:29:28.076 INFO:tasks.rados.rados.0.smithi026.stdout:3079: read oid 13 snap 285 2024-09-13T14:29:28.076 INFO:tasks.rados.rados.0.smithi026.stdout:3079: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:29:28.076 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 296 2024-09-13T14:29:28.076 INFO:tasks.rados.rados.0.smithi026.stdout:3080: seq_num 955 ranges {0=802816,802816=557056,1359872=245760} 2024-09-13T14:29:28.080 INFO:tasks.rados.rados.0.smithi026.stdout:3080: writing smithi02683976-33 from 0 to 802816 tid 1 2024-09-13T14:29:28.085 INFO:tasks.rados.rados.0.smithi026.stdout:3080: writing smithi02683976-33 from 802816 to 1359872 tid 2 2024-09-13T14:29:28.087 INFO:tasks.rados.rados.0.smithi026.stdout:3080: writing smithi02683976-33 from 1359872 to 1605632 tid 3 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 296 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3074: finishing write tid 4 to smithi02683976-4 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3074: finishing write tid 5 to smithi02683976-4 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1312 (ObjNum 953 snap 296 seq_num 953) dirty exists 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3074: left oid 4 (ObjNum 953 snap 296 seq_num 953) 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3076: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3076: finishing copy_from to smithi02683976-21 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1184 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:28.088 INFO:tasks.rados.rados.0.smithi026.stdout:3081: seq_num 956 ranges {1236992=417792,1654784=434176,2088960=770048} 2024-09-13T14:29:28.098 INFO:tasks.rados.rados.0.smithi026.stdout:3081: writing smithi02683976-42 from 1236992 to 1654784 tid 1 2024-09-13T14:29:28.101 INFO:tasks.rados.rados.0.smithi026.stdout:3081: writing smithi02683976-42 from 1654784 to 2088960 tid 2 2024-09-13T14:29:28.107 INFO:tasks.rados.rados.0.smithi026.stdout:3081: writing smithi02683976-42 from 2088960 to 2859008 tid 3 2024-09-13T14:29:28.107 INFO:tasks.rados.rados.0.smithi026.stdout:3074: done (4 left) 2024-09-13T14:29:28.107 INFO:tasks.rados.rados.0.smithi026.stdout:3076: done (3 left) 2024-09-13T14:29:28.107 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 296 2024-09-13T14:29:28.107 INFO:tasks.rados.rados.0.smithi026.stdout:3082: seq_num 957 ranges {2367488=434176,2801664=720896,3522560=483328} 2024-09-13T14:29:28.122 INFO:tasks.rados.rados.0.smithi026.stdout:3082: writing smithi02683976-40 from 2367488 to 2801664 tid 1 2024-09-13T14:29:28.127 INFO:tasks.rados.rados.0.smithi026.stdout:3082: writing smithi02683976-40 from 2801664 to 3522560 tid 2 2024-09-13T14:29:28.130 INFO:tasks.rados.rados.0.smithi026.stdout:3082: writing smithi02683976-40 from 3522560 to 4005888 tid 3 2024-09-13T14:29:28.130 INFO:tasks.rados.rados.0.smithi026.stdout:3083: delete oid 37 current snap is 296 2024-09-13T14:29:28.130 INFO:tasks.rados.rados.0.smithi026.stdout:3079: expect (ObjNum 893 snap 277 seq_num 893) 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: finishing write tid 1 to smithi02683976-33 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: finishing write tid 2 to smithi02683976-33 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: finishing write tid 3 to smithi02683976-33 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: finishing write tid 4 to smithi02683976-33 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3081: finishing write tid 1 to smithi02683976-42 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: finishing write tid 5 to smithi02683976-33 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1825 (ObjNum 955 snap 296 seq_num 955) dirty exists 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: left oid 33 (ObjNum 955 snap 296 seq_num 955) 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3081: finishing write tid 2 to smithi02683976-42 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3079: done (4 left) 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3080: done (3 left) 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3083: done (2 left) 2024-09-13T14:29:28.147 INFO:tasks.rados.rados.0.smithi026.stdout:3084: snap_create 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3081: finishing write tid 3 to smithi02683976-42 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3081: finishing write tid 4 to smithi02683976-42 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3081: finishing write tid 5 to smithi02683976-42 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1459 (ObjNum 956 snap 296 seq_num 956) dirty exists 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3081: left oid 42 (ObjNum 956 snap 296 seq_num 956) 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: finishing write tid 1 to smithi02683976-40 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: finishing write tid 2 to smithi02683976-40 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: finishing write tid 3 to smithi02683976-40 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: finishing write tid 4 to smithi02683976-40 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: finishing write tid 5 to smithi02683976-40 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1210 (ObjNum 957 snap 296 seq_num 957) dirty exists 2024-09-13T14:29:28.148 INFO:tasks.rados.rados.0.smithi026.stdout:3082: left oid 40 (ObjNum 957 snap 296 seq_num 957) 2024-09-13T14:29:28.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:28 smithi026 ceph-mon[97022]: pgmap v596: 121 pgs: 121 active+clean; 516 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 18 op/s 2024-09-13T14:29:28.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:28 smithi026 ceph-mon[97022]: osdmap e663: 8 total, 8 up, 8 in 2024-09-13T14:29:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:28 smithi026 ceph-mon[99461]: pgmap v596: 121 pgs: 121 active+clean; 516 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 18 op/s 2024-09-13T14:29:28.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:28 smithi026 ceph-mon[99461]: osdmap e663: 8 total, 8 up, 8 in 2024-09-13T14:29:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:28 smithi145 ceph-mon[89145]: pgmap v596: 121 pgs: 121 active+clean; 516 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 18 op/s 2024-09-13T14:29:28.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:28 smithi145 ceph-mon[89145]: osdmap e663: 8 total, 8 up, 8 in 2024-09-13T14:29:28.994 INFO:tasks.rados.rados.0.smithi026.stdout:3081: done (2 left) 2024-09-13T14:29:28.994 INFO:tasks.rados.rados.0.smithi026.stdout:3082: done (1 left) 2024-09-13T14:29:28.994 INFO:tasks.rados.rados.0.smithi026.stdout:3084: done (0 left) 2024-09-13T14:29:28.994 INFO:tasks.rados.rados.0.smithi026.stdout:3085: snap_create 2024-09-13T14:29:29.337 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)" 2024-09-13T14:29:29.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:29 smithi026 ceph-mon[99461]: osdmap e664: 8 total, 8 up, 8 in 2024-09-13T14:29:29.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:29 smithi026 ceph-mon[99461]: osdmap e665: 8 total, 8 up, 8 in 2024-09-13T14:29:29.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:29 smithi026 ceph-mon[97022]: osdmap e664: 8 total, 8 up, 8 in 2024-09-13T14:29:29.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:29 smithi026 ceph-mon[97022]: osdmap e665: 8 total, 8 up, 8 in 2024-09-13T14:29:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:29 smithi145 ceph-mon[89145]: osdmap e664: 8 total, 8 up, 8 in 2024-09-13T14:29:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:29 smithi145 ceph-mon[89145]: osdmap e665: 8 total, 8 up, 8 in 2024-09-13T14:29:29.995 INFO:tasks.rados.rados.0.smithi026.stdout:3085: done (0 left) 2024-09-13T14:29:29.996 INFO:tasks.rados.rados.0.smithi026.stdout:3086: snap_create 2024-09-13T14:29:30.015 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:29:30.016 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-13T14:29:30.016 DEBUG:teuthology.parallel:result is None 2024-09-13T14:29:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[97022]: pgmap v599: 121 pgs: 121 active+clean; 518 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-13T14:29:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/573725318' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[97022]: osdmap e666: 8 total, 8 up, 8 in 2024-09-13T14:29:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[99461]: pgmap v599: 121 pgs: 121 active+clean; 518 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-13T14:29:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/573725318' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:30 smithi026 ceph-mon[99461]: osdmap e666: 8 total, 8 up, 8 in 2024-09-13T14:29:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:30 smithi145 ceph-mon[89145]: pgmap v599: 121 pgs: 121 active+clean; 518 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-13T14:29:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:30 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/573725318' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:30 smithi145 ceph-mon[89145]: osdmap e666: 8 total, 8 up, 8 in 2024-09-13T14:29:30.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:30.997 INFO:tasks.rados.rados.0.smithi026.stdout:3086: done (0 left) 2024-09-13T14:29:30.998 INFO:tasks.rados.rados.0.smithi026.stdout:3087: snap_create 2024-09-13T14:29:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:31] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.51.0" 2024-09-13T14:29:31.999 INFO:tasks.rados.rados.0.smithi026.stdout:3087: done (0 left) 2024-09-13T14:29:31.999 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 300 2024-09-13T14:29:31.999 INFO:tasks.rados.rados.0.smithi026.stdout:3088: seq_num 959 ranges {1523712=466944,1990656=540672,2531328=557056,3088384=106496} 2024-09-13T14:29:32.010 INFO:tasks.rados.rados.0.smithi026.stdout:3088: writing smithi02683976-17 from 1523712 to 1990656 tid 1 2024-09-13T14:29:32.013 INFO:tasks.rados.rados.0.smithi026.stdout:3088: writing smithi02683976-17 from 1990656 to 2531328 tid 2 2024-09-13T14:29:32.016 INFO:tasks.rados.rados.0.smithi026.stdout:3088: writing smithi02683976-17 from 2531328 to 3088384 tid 3 2024-09-13T14:29:32.017 INFO:tasks.rados.rados.0.smithi026.stdout:3088: writing smithi02683976-17 from 3088384 to 3194880 tid 4 2024-09-13T14:29:32.017 INFO:tasks.rados.rados.0.smithi026.stdout:3089: rollback oid 40 current snap is 300 2024-09-13T14:29:32.017 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 40 to 288 2024-09-13T14:29:32.018 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 1 to smithi02683976-17 2024-09-13T14:29:32.018 INFO:tasks.rados.rados.0.smithi026.stdout:3090: snap_remove snap 296 2024-09-13T14:29:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:31 smithi026 ceph-mon[97022]: pgmap v602: 121 pgs: 121 active+clean; 548 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:31 smithi026 ceph-mon[97022]: osdmap e667: 8 total, 8 up, 8 in 2024-09-13T14:29:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:32 smithi026 ceph-mon[99461]: pgmap v602: 121 pgs: 121 active+clean; 548 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:32 smithi026 ceph-mon[99461]: osdmap e667: 8 total, 8 up, 8 in 2024-09-13T14:29:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:32 smithi145 ceph-mon[89145]: pgmap v602: 121 pgs: 121 active+clean; 548 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:32 smithi145 ceph-mon[89145]: osdmap e667: 8 total, 8 up, 8 in 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3090: done (2 left) 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3091: read oid 29 snap 299 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3091: expect deleted 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 300 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 2 to smithi02683976-17 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 3 to smithi02683976-17 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 4 to smithi02683976-17 2024-09-13T14:29:33.012 INFO:tasks.rados.rados.0.smithi026.stdout:3089: finishing rollback tid 0 to smithi02683976-40 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 5 to smithi02683976-17 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3089: finishing rollback tid 1 to smithi02683976-40 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3089: finishing rollback tid 2 to smithi02683976-40 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1213 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3088: finishing write tid 6 to smithi02683976-17 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1206 (ObjNum 959 snap 300 seq_num 959) dirty exists 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3088: left oid 17 (ObjNum 959 snap 300 seq_num 959) 2024-09-13T14:29:33.013 INFO:tasks.rados.rados.0.smithi026.stdout:3092: seq_num 960 ranges {2351104=483328,2834432=425984,3260416=761856,4022272=16384} 2024-09-13T14:29:33.027 INFO:tasks.rados.rados.0.smithi026.stdout:3092: writing smithi02683976-19 from 2351104 to 2834432 tid 1 2024-09-13T14:29:33.030 INFO:tasks.rados.rados.0.smithi026.stdout:3092: writing smithi02683976-19 from 2834432 to 3260416 tid 2 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3092: writing smithi02683976-19 from 3260416 to 4022272 tid 3 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3092: writing smithi02683976-19 from 4022272 to 4038656 tid 4 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3088: done (3 left) 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3089: done (2 left) 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3093: rollback oid 8 current snap is 300 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 288 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 1 to smithi02683976-19 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 2 to smithi02683976-19 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3091: done (2 left) 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:3094: rollback oid 1 current snap is 300 2024-09-13T14:29:33.036 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 297 2024-09-13T14:29:33.037 INFO:tasks.rados.rados.0.smithi026.stdout:3095: snap_create 2024-09-13T14:29:33.042 INFO:tasks.rados.rados.0.smithi026.stdout:3093: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:29:33.042 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 3 to smithi02683976-19 2024-09-13T14:29:33.042 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 4 to smithi02683976-19 2024-09-13T14:29:33.043 INFO:tasks.rados.rados.0.smithi026.stdout:3094: finishing rollback tid 0 to smithi02683976-1 2024-09-13T14:29:33.043 INFO:tasks.rados.rados.0.smithi026.stdout:3093: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:29:33.043 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 5 to smithi02683976-19 2024-09-13T14:29:33.044 INFO:tasks.rados.rados.0.smithi026.stdout:3094: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:29:33.044 INFO:tasks.rados.rados.0.smithi026.stdout:3094: finishing rollback tid 2 to smithi02683976-1 2024-09-13T14:29:33.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1835 (ObjNum 893 snap 277 seq_num 893) dirty exists 2024-09-13T14:29:33.045 INFO:tasks.rados.rados.0.smithi026.stdout:3093: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:29:33.045 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1292 (ObjNum 860 snap 266 seq_num 860) dirty exists 2024-09-13T14:29:33.046 INFO:tasks.rados.rados.0.smithi026.stdout:3092: finishing write tid 6 to smithi02683976-19 2024-09-13T14:29:33.046 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1617 (ObjNum 960 snap 300 seq_num 960) dirty exists 2024-09-13T14:29:33.046 INFO:tasks.rados.rados.0.smithi026.stdout:3092: left oid 19 (ObjNum 960 snap 300 seq_num 960) 2024-09-13T14:29:33.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:33 smithi026 ceph-mon[97022]: osdmap e668: 8 total, 8 up, 8 in 2024-09-13T14:29:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:33 smithi026 ceph-mon[99461]: osdmap e668: 8 total, 8 up, 8 in 2024-09-13T14:29:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:33 smithi145 ceph-mon[89145]: osdmap e668: 8 total, 8 up, 8 in 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:3092: done (3 left) 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:3093: done (2 left) 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:3094: done (1 left) 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:3095: done (0 left) 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 301 2024-09-13T14:29:33.996 INFO:tasks.rados.rados.0.smithi026.stdout:3096: seq_num 961 ranges {2334720=499712,2834432=712704,3547136=491520} 2024-09-13T14:29:34.011 INFO:tasks.rados.rados.0.smithi026.stdout:3096: writing smithi02683976-30 from 2334720 to 2834432 tid 1 2024-09-13T14:29:34.015 INFO:tasks.rados.rados.0.smithi026.stdout:3096: writing smithi02683976-30 from 2834432 to 3547136 tid 2 2024-09-13T14:29:34.018 INFO:tasks.rados.rados.0.smithi026.stdout:3096: writing smithi02683976-30 from 3547136 to 4038656 tid 3 2024-09-13T14:29:34.019 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 301 2024-09-13T14:29:34.019 INFO:tasks.rados.rados.0.smithi026.stdout:3097: seq_num 962 ranges {1679360=516096,2195456=589824,2785280=614400} 2024-09-13T14:29:34.030 INFO:tasks.rados.rados.0.smithi026.stdout:3097: writing smithi02683976-38 from 1679360 to 2195456 tid 1 2024-09-13T14:29:34.034 INFO:tasks.rados.rados.0.smithi026.stdout:3097: writing smithi02683976-38 from 2195456 to 2785280 tid 2 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3097: writing smithi02683976-38 from 2785280 to 3399680 tid 3 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3098: snap_create 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: finishing write tid 1 to smithi02683976-30 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: finishing write tid 2 to smithi02683976-30 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: finishing write tid 3 to smithi02683976-30 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: finishing write tid 4 to smithi02683976-30 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: finishing write tid 5 to smithi02683976-30 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1316 (ObjNum 961 snap 301 seq_num 961) dirty exists 2024-09-13T14:29:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3096: left oid 30 (ObjNum 961 snap 301 seq_num 961) 2024-09-13T14:29:34.040 INFO:tasks.rados.rados.0.smithi026.stdout:3097: finishing write tid 1 to smithi02683976-38 2024-09-13T14:29:34.040 INFO:tasks.rados.rados.0.smithi026.stdout:3097: finishing write tid 2 to smithi02683976-38 2024-09-13T14:29:34.042 INFO:tasks.rados.rados.0.smithi026.stdout:3097: finishing write tid 3 to smithi02683976-38 2024-09-13T14:29:34.042 INFO:tasks.rados.rados.0.smithi026.stdout:3097: finishing write tid 4 to smithi02683976-38 2024-09-13T14:29:34.044 INFO:tasks.rados.rados.0.smithi026.stdout:3097: finishing write tid 5 to smithi02683976-38 2024-09-13T14:29:34.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1482 (ObjNum 962 snap 301 seq_num 962) dirty exists 2024-09-13T14:29:34.044 INFO:tasks.rados.rados.0.smithi026.stdout:3097: left oid 38 (ObjNum 962 snap 301 seq_num 962) 2024-09-13T14:29:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[99461]: pgmap v605: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[99461]: osdmap e669: 8 total, 8 up, 8 in 2024-09-13T14:29:34.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[99461]: osdmap e670: 8 total, 8 up, 8 in 2024-09-13T14:29:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[97022]: pgmap v605: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[97022]: osdmap e669: 8 total, 8 up, 8 in 2024-09-13T14:29:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:34 smithi026 ceph-mon[97022]: osdmap e670: 8 total, 8 up, 8 in 2024-09-13T14:29:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:34 smithi145 ceph-mon[89145]: pgmap v605: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 22 op/s 2024-09-13T14:29:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:34 smithi145 ceph-mon[89145]: osdmap e669: 8 total, 8 up, 8 in 2024-09-13T14:29:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:34 smithi145 ceph-mon[89145]: osdmap e670: 8 total, 8 up, 8 in 2024-09-13T14:29:35.016 INFO:tasks.rados.rados.0.smithi026.stdout:3096: done (2 left) 2024-09-13T14:29:35.016 INFO:tasks.rados.rados.0.smithi026.stdout:3097: done (1 left) 2024-09-13T14:29:35.016 INFO:tasks.rados.rados.0.smithi026.stdout:3098: done (0 left) 2024-09-13T14:29:35.016 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 302 2024-09-13T14:29:35.016 INFO:tasks.rados.rados.0.smithi026.stdout:3099: seq_num 963 ranges {0=532480,532480=475136,1007616=532480,1540096=196608} 2024-09-13T14:29:35.018 INFO:tasks.rados.rados.0.smithi026.stdout:3099: writing smithi02683976-31 from 0 to 532480 tid 1 2024-09-13T14:29:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3099: writing smithi02683976-31 from 532480 to 1007616 tid 2 2024-09-13T14:29:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3099: writing smithi02683976-31 from 1007616 to 1540096 tid 3 2024-09-13T14:29:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:3099: writing smithi02683976-31 from 1540096 to 1736704 tid 4 2024-09-13T14:29:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:3100: rmattr oid 35 current snap is 302 2024-09-13T14:29:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:3100: done (1 left) 2024-09-13T14:29:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:3101: read oid 29 snap 301 2024-09-13T14:29:35.028 INFO:tasks.rados.rados.0.smithi026.stdout:3101: expect deleted 2024-09-13T14:29:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:3102: copy_from oid 11 from oid 13 current snap is 302 2024-09-13T14:29:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 302 2024-09-13T14:29:35.029 INFO:tasks.rados.rados.0.smithi026.stdout:3103: seq_num 964 ranges {4079616=548864,4628480=761856,5390336=442368} 2024-09-13T14:29:35.052 INFO:tasks.rados.rados.0.smithi026.stdout:3103: writing smithi02683976-46 from 4079616 to 4628480 tid 1 2024-09-13T14:29:35.056 INFO:tasks.rados.rados.0.smithi026.stdout:3103: writing smithi02683976-46 from 4628480 to 5390336 tid 2 2024-09-13T14:29:35.059 INFO:tasks.rados.rados.0.smithi026.stdout:3103: writing smithi02683976-46 from 5390336 to 5832704 tid 3 2024-09-13T14:29:35.059 INFO:tasks.rados.rados.0.smithi026.stdout:3104: delete oid 18 current snap is 302 2024-09-13T14:29:35.059 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 1 to smithi02683976-31 2024-09-13T14:29:35.059 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 2 to smithi02683976-31 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 3 to smithi02683976-31 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 4 to smithi02683976-31 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 5 to smithi02683976-31 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3099: finishing write tid 6 to smithi02683976-31 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1533 (ObjNum 963 snap 302 seq_num 963) dirty exists 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3099: left oid 31 (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3102: finishing copy_from racing read to smithi02683976-11 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3102: finishing copy_from to smithi02683976-11 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1610 (ObjNum 948 snap 296 seq_num 948) dirty exists 2024-09-13T14:29:35.060 INFO:tasks.rados.rados.0.smithi026.stdout:3103: finishing write tid 1 to smithi02683976-46 2024-09-13T14:29:35.061 INFO:tasks.rados.rados.0.smithi026.stdout:3103: finishing write tid 2 to smithi02683976-46 2024-09-13T14:29:35.062 INFO:tasks.rados.rados.0.smithi026.stdout:3103: finishing write tid 3 to smithi02683976-46 2024-09-13T14:29:35.063 INFO:tasks.rados.rados.0.smithi026.stdout:3103: finishing write tid 4 to smithi02683976-46 2024-09-13T14:29:35.063 INFO:tasks.rados.rados.0.smithi026.stdout:3099: done (4 left) 2024-09-13T14:29:35.063 INFO:tasks.rados.rados.0.smithi026.stdout:3101: done (3 left) 2024-09-13T14:29:35.063 INFO:tasks.rados.rados.0.smithi026.stdout:3102: done (2 left) 2024-09-13T14:29:35.063 INFO:tasks.rados.rados.0.smithi026.stdout:3104: done (1 left) 2024-09-13T14:29:35.064 INFO:tasks.rados.rados.0.smithi026.stdout:3105: read oid 33 snap -1 2024-09-13T14:29:35.064 INFO:tasks.rados.rados.0.smithi026.stdout:3105: expect (ObjNum 955 snap 296 seq_num 955) 2024-09-13T14:29:35.064 INFO:tasks.rados.rados.0.smithi026.stdout:3106: copy_from oid 39 from oid 48 current snap is 302 2024-09-13T14:29:35.064 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 302 2024-09-13T14:29:35.064 INFO:tasks.rados.rados.0.smithi026.stdout:3107: seq_num 966 ranges {9199616=581632,9781248=581632,10362880=622592} 2024-09-13T14:29:35.117 INFO:tasks.rados.rados.0.smithi026.stdout:3107: writing smithi02683976-8 from 9199616 to 9781248 tid 1 2024-09-13T14:29:35.120 INFO:tasks.rados.rados.0.smithi026.stdout:3107: writing smithi02683976-8 from 9781248 to 10362880 tid 2 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3107: writing smithi02683976-8 from 10362880 to 10985472 tid 3 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3108: read oid 24 snap -1 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3108: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3109: read oid 34 snap -1 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3109: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 302 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3103: finishing write tid 5 to smithi02683976-46 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1431 (ObjNum 964 snap 302 seq_num 964) dirty exists 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3103: left oid 46 (ObjNum 964 snap 302 seq_num 964) 2024-09-13T14:29:35.125 INFO:tasks.rados.rados.0.smithi026.stdout:3106: finishing copy_from to smithi02683976-39 2024-09-13T14:29:35.126 INFO:tasks.rados.rados.0.smithi026.stdout:3106: got expected ENOENT (src dne) 2024-09-13T14:29:35.126 INFO:tasks.rados.rados.0.smithi026.stdout:3106: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:29:35.126 INFO:tasks.rados.rados.0.smithi026.stdout:3105: expect (ObjNum 955 snap 296 seq_num 955) 2024-09-13T14:29:35.171 INFO:tasks.rados.rados.0.smithi026.stdout:3107: finishing write tid 1 to smithi02683976-8 2024-09-13T14:29:35.171 INFO:tasks.rados.rados.0.smithi026.stdout:3107: finishing write tid 2 to smithi02683976-8 2024-09-13T14:29:35.171 INFO:tasks.rados.rados.0.smithi026.stdout:3110: seq_num 967 ranges {565248=598016,1163264=466944,1630208=516096,2146304=221184} 2024-09-13T14:29:35.177 INFO:tasks.rados.rados.0.smithi026.stdout:3110: writing smithi02683976-1 from 565248 to 1163264 tid 1 2024-09-13T14:29:35.180 INFO:tasks.rados.rados.0.smithi026.stdout:3110: writing smithi02683976-1 from 1163264 to 1630208 tid 2 2024-09-13T14:29:35.183 INFO:tasks.rados.rados.0.smithi026.stdout:3110: writing smithi02683976-1 from 1630208 to 2146304 tid 3 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3110: writing smithi02683976-1 from 2146304 to 2367488 tid 4 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3103: done (6 left) 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3105: done (5 left) 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3106: done (4 left) 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3111: rmattr oid 6 current snap is 302 2024-09-13T14:29:35.185 INFO:tasks.rados.rados.0.smithi026.stdout:3111: done (4 left) 2024-09-13T14:29:35.186 INFO:tasks.rados.rados.0.smithi026.stdout:3112: rollback oid 10 current snap is 302 2024-09-13T14:29:35.186 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 290 2024-09-13T14:29:35.186 INFO:tasks.rados.rados.0.smithi026.stdout:3107: finishing write tid 3 to smithi02683976-8 2024-09-13T14:29:35.186 INFO:tasks.rados.rados.0.smithi026.stdout:3107: finishing write tid 4 to smithi02683976-8 2024-09-13T14:29:35.186 INFO:tasks.rados.rados.0.smithi026.stdout:3108: expect (ObjNum 904 snap 280 seq_num 904) 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3107: finishing write tid 5 to smithi02683976-8 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1296 (ObjNum 966 snap 302 seq_num 966) dirty exists 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3107: left oid 8 (ObjNum 966 snap 302 seq_num 966) 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3107: done (4 left) 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3108: done (3 left) 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3113: copy_from oid 8 from oid 6 current snap is 302 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 1 to smithi02683976-1 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 2 to smithi02683976-1 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3112: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:29:35.279 INFO:tasks.rados.rados.0.smithi026.stdout:3112: finishing rollback tid 2 to smithi02683976-10 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1215 (ObjNum 835 snap 260 seq_num 835) dirty exists 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 3 to smithi02683976-1 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 4 to smithi02683976-1 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 5 to smithi02683976-1 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3110: finishing write tid 6 to smithi02683976-1 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1841 (ObjNum 967 snap 302 seq_num 967) dirty exists 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3110: left oid 1 (ObjNum 967 snap 302 seq_num 967) 2024-09-13T14:29:35.280 INFO:tasks.rados.rados.0.smithi026.stdout:3109: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:36.275 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:36 smithi145 ceph-mon[89145]: pgmap v608: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 362 KiB/s wr, 3 op/s 2024-09-13T14:29:36.275 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:36 smithi145 ceph-mon[89145]: osdmap e671: 8 total, 8 up, 8 in 2024-09-13T14:29:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:36 smithi026 ceph-mon[97022]: pgmap v608: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 362 KiB/s wr, 3 op/s 2024-09-13T14:29:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:36 smithi026 ceph-mon[97022]: osdmap e671: 8 total, 8 up, 8 in 2024-09-13T14:29:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:36 smithi026 ceph-mon[99461]: pgmap v608: 121 pgs: 121 active+clean; 549 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 362 KiB/s wr, 3 op/s 2024-09-13T14:29:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:36 smithi026 ceph-mon[99461]: osdmap e671: 8 total, 8 up, 8 in 2024-09-13T14:29:36.952 INFO:tasks.rados.rados.0.smithi026.stdout:3113: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:29:36.952 INFO:tasks.rados.rados.0.smithi026.stdout:3113: finishing copy_from to smithi02683976-8 2024-09-13T14:29:36.952 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1516 (ObjNum 943 snap 294 seq_num 943) dirty exists 2024-09-13T14:29:36.953 INFO:tasks.rados.rados.0.smithi026.stdout:3109: done (3 left) 2024-09-13T14:29:36.953 INFO:tasks.rados.rados.0.smithi026.stdout:3110: done (2 left) 2024-09-13T14:29:36.953 INFO:tasks.rados.rados.0.smithi026.stdout:3112: done (1 left) 2024-09-13T14:29:36.953 INFO:tasks.rados.rados.0.smithi026.stdout:3113: done (0 left) 2024-09-13T14:29:36.953 INFO:tasks.rados.rados.0.smithi026.stdout:3114: read oid 11 snap 298 2024-09-13T14:29:36.954 INFO:tasks.rados.rados.0.smithi026.stdout:3114: expect (ObjNum 916 snap 286 seq_num 916) 2024-09-13T14:29:36.954 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 302 2024-09-13T14:29:36.954 INFO:tasks.rados.rados.0.smithi026.stdout:3115: seq_num 968 ranges {14024704=614400,14639104=745472,15384576=458752} 2024-09-13T14:29:37.032 INFO:tasks.rados.rados.0.smithi026.stdout:3115: writing smithi02683976-14 from 14024704 to 14639104 tid 1 2024-09-13T14:29:37.037 INFO:tasks.rados.rados.0.smithi026.stdout:3115: writing smithi02683976-14 from 14639104 to 15384576 tid 2 2024-09-13T14:29:37.039 INFO:tasks.rados.rados.0.smithi026.stdout:3115: writing smithi02683976-14 from 15384576 to 15843328 tid 3 2024-09-13T14:29:37.039 INFO:tasks.rados.rados.0.smithi026.stdout:3116: delete oid 33 current snap is 302 2024-09-13T14:29:37.039 INFO:tasks.rados.rados.0.smithi026.stdout:3114: expect (ObjNum 916 snap 286 seq_num 916) 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:3115: finishing write tid 1 to smithi02683976-14 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:3115: finishing write tid 2 to smithi02683976-14 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:3115: finishing write tid 3 to smithi02683976-14 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:3115: finishing write tid 4 to smithi02683976-14 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:3115: finishing write tid 5 to smithi02683976-14 2024-09-13T14:29:37.066 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1489 (ObjNum 968 snap 302 seq_num 968) dirty exists 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:3115: left oid 14 (ObjNum 968 snap 302 seq_num 968) 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:3114: done (2 left) 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:3115: done (1 left) 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:3116: done (0 left) 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 302 2024-09-13T14:29:37.067 INFO:tasks.rados.rados.0.smithi026.stdout:3117: seq_num 970 ranges {3399680=655360,4055040=516096,4571136=688128} 2024-09-13T14:29:37.088 INFO:tasks.rados.rados.0.smithi026.stdout:3117: writing smithi02683976-38 from 3399680 to 4055040 tid 1 2024-09-13T14:29:37.091 INFO:tasks.rados.rados.0.smithi026.stdout:3117: writing smithi02683976-38 from 4055040 to 4571136 tid 2 2024-09-13T14:29:37.095 INFO:tasks.rados.rados.0.smithi026.stdout:3117: writing smithi02683976-38 from 4571136 to 5259264 tid 3 2024-09-13T14:29:37.095 INFO:tasks.rados.rados.0.smithi026.stdout:3118: snap_create 2024-09-13T14:29:37.095 INFO:tasks.rados.rados.0.smithi026.stdout:3117: finishing write tid 1 to smithi02683976-38 2024-09-13T14:29:37.095 INFO:tasks.rados.rados.0.smithi026.stdout:3117: finishing write tid 2 to smithi02683976-38 2024-09-13T14:29:37.097 INFO:tasks.rados.rados.0.smithi026.stdout:3117: finishing write tid 3 to smithi02683976-38 2024-09-13T14:29:37.097 INFO:tasks.rados.rados.0.smithi026.stdout:3117: finishing write tid 4 to smithi02683976-38 2024-09-13T14:29:37.098 INFO:tasks.rados.rados.0.smithi026.stdout:3117: finishing write tid 5 to smithi02683976-38 2024-09-13T14:29:37.098 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1494 (ObjNum 970 snap 302 seq_num 970) dirty exists 2024-09-13T14:29:37.098 INFO:tasks.rados.rados.0.smithi026.stdout:3117: left oid 38 (ObjNum 970 snap 302 seq_num 970) 2024-09-13T14:29:38.023 INFO:tasks.rados.rados.0.smithi026.stdout:3117: done (1 left) 2024-09-13T14:29:38.023 INFO:tasks.rados.rados.0.smithi026.stdout:3118: done (0 left) 2024-09-13T14:29:38.024 INFO:tasks.rados.rados.0.smithi026.stdout:3119: copy_from oid 40 from oid 20 current snap is 303 2024-09-13T14:29:38.024 INFO:tasks.rados.rados.0.smithi026.stdout:3120: rmattr oid 46 current snap is 303 2024-09-13T14:29:38.024 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 303 2024-09-13T14:29:38.024 INFO:tasks.rados.rados.0.smithi026.stdout:3121: seq_num 971 ranges {1482752=671744,2154496=802816,2957312=401408} 2024-09-13T14:29:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3121: writing smithi02683976-11 from 1482752 to 2154496 tid 1 2024-09-13T14:29:38.040 INFO:tasks.rados.rados.0.smithi026.stdout:3121: writing smithi02683976-11 from 2154496 to 2957312 tid 2 2024-09-13T14:29:38.042 INFO:tasks.rados.rados.0.smithi026.stdout:3121: writing smithi02683976-11 from 2957312 to 3358720 tid 3 2024-09-13T14:29:38.042 INFO:tasks.rados.rados.0.smithi026.stdout:3122: snap_create 2024-09-13T14:29:38.043 INFO:tasks.rados.rados.0.smithi026.stdout:3119: finishing copy_from to smithi02683976-40 2024-09-13T14:29:38.043 INFO:tasks.rados.rados.0.smithi026.stdout:3119: got expected ENOENT (src dne) 2024-09-13T14:29:38.043 INFO:tasks.rados.rados.0.smithi026.stdout:3119: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:29:38.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1432 (ObjNum 964 snap 302 seq_num 964) dirty exists 2024-09-13T14:29:38.043 INFO:tasks.rados.rados.0.smithi026.stdout:3121: finishing write tid 1 to smithi02683976-11 2024-09-13T14:29:38.046 INFO:tasks.rados.rados.0.smithi026.stdout:3121: finishing write tid 2 to smithi02683976-11 2024-09-13T14:29:38.046 INFO:tasks.rados.rados.0.smithi026.stdout:3121: finishing write tid 3 to smithi02683976-11 2024-09-13T14:29:38.046 INFO:tasks.rados.rados.0.smithi026.stdout:3121: finishing write tid 4 to smithi02683976-11 2024-09-13T14:29:38.047 INFO:tasks.rados.rados.0.smithi026.stdout:3121: finishing write tid 5 to smithi02683976-11 2024-09-13T14:29:38.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1614 (ObjNum 971 snap 303 seq_num 971) dirty exists 2024-09-13T14:29:38.047 INFO:tasks.rados.rados.0.smithi026.stdout:3121: left oid 11 (ObjNum 971 snap 303 seq_num 971) 2024-09-13T14:29:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:38 smithi026 ceph-mon[97022]: pgmap v610: 121 pgs: 121 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-13T14:29:38.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:38 smithi026 ceph-mon[99461]: pgmap v610: 121 pgs: 121 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-13T14:29:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:38 smithi145 ceph-mon[89145]: pgmap v610: 121 pgs: 121 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.3 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:3119: done (3 left) 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:3120: done (2 left) 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:3121: done (1 left) 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:3122: done (0 left) 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 39 current snap is 304 2024-09-13T14:29:38.998 INFO:tasks.rados.rados.0.smithi026.stdout:3123: seq_num 972 ranges {1048576=688128,1736704=688128,2424832=516096} 2024-09-13T14:29:39.007 INFO:tasks.rados.rados.0.smithi026.stdout:3123: writing smithi02683976-39 from 1048576 to 1736704 tid 1 2024-09-13T14:29:39.011 INFO:tasks.rados.rados.0.smithi026.stdout:3123: writing smithi02683976-39 from 1736704 to 2424832 tid 2 2024-09-13T14:29:39.015 INFO:tasks.rados.rados.0.smithi026.stdout:3123: writing smithi02683976-39 from 2424832 to 2940928 tid 3 2024-09-13T14:29:39.016 INFO:tasks.rados.rados.0.smithi026.stdout:3124: read oid 17 snap -1 2024-09-13T14:29:39.016 INFO:tasks.rados.rados.0.smithi026.stdout:3124: expect (ObjNum 959 snap 300 seq_num 959) 2024-09-13T14:29:39.016 INFO:tasks.rados.rados.0.smithi026.stdout:3125: snap_create 2024-09-13T14:29:39.020 INFO:tasks.rados.rados.0.smithi026.stdout:3123: finishing write tid 1 to smithi02683976-39 2024-09-13T14:29:39.021 INFO:tasks.rados.rados.0.smithi026.stdout:3123: finishing write tid 2 to smithi02683976-39 2024-09-13T14:29:39.021 INFO:tasks.rados.rados.0.smithi026.stdout:3123: finishing write tid 3 to smithi02683976-39 2024-09-13T14:29:39.022 INFO:tasks.rados.rados.0.smithi026.stdout:3123: finishing write tid 4 to smithi02683976-39 2024-09-13T14:29:39.023 INFO:tasks.rados.rados.0.smithi026.stdout:3123: finishing write tid 5 to smithi02683976-39 2024-09-13T14:29:39.023 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1280 (ObjNum 972 snap 304 seq_num 972) dirty exists 2024-09-13T14:29:39.023 INFO:tasks.rados.rados.0.smithi026.stdout:3123: left oid 39 (ObjNum 972 snap 304 seq_num 972) 2024-09-13T14:29:39.037 INFO:tasks.rados.rados.0.smithi026.stdout:3124: expect (ObjNum 959 snap 300 seq_num 959) 2024-09-13T14:29:39.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:39 smithi026 ceph-mon[97022]: osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:29:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:39 smithi026 ceph-mon[97022]: osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:29:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:39 smithi026 ceph-mon[99461]: osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:29:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:39 smithi026 ceph-mon[99461]: osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:29:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:39 smithi145 ceph-mon[89145]: osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:29:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:39 smithi145 ceph-mon[89145]: osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:29:40.000 INFO:tasks.rados.rados.0.smithi026.stdout:3123: done (2 left) 2024-09-13T14:29:40.000 INFO:tasks.rados.rados.0.smithi026.stdout:3124: done (1 left) 2024-09-13T14:29:40.000 INFO:tasks.rados.rados.0.smithi026.stdout:3125: done (0 left) 2024-09-13T14:29:40.000 INFO:tasks.rados.rados.0.smithi026.stdout:3126: snap_create 2024-09-13T14:29:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:40 smithi026 ceph-mon[97022]: pgmap v612: 121 pgs: 121 active+clean; 557 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:29:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:40 smithi026 ceph-mon[97022]: osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:29:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:40 smithi026 ceph-mon[99461]: pgmap v612: 121 pgs: 121 active+clean; 557 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:29:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:40 smithi026 ceph-mon[99461]: osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:29:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:40 smithi145 ceph-mon[89145]: pgmap v612: 121 pgs: 121 active+clean; 557 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:29:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:40 smithi145 ceph-mon[89145]: osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:29:40.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:3126: done (0 left) 2024-09-13T14:29:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:3127: rollback oid 32 current snap is 306 2024-09-13T14:29:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 290 2024-09-13T14:29:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:3128: copy_from oid 1 from oid 15 current snap is 306 2024-09-13T14:29:41.001 INFO:tasks.rados.rados.0.smithi026.stdout:3129: snap_remove snap 297 2024-09-13T14:29:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:41] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.51.0" 2024-09-13T14:29:42.003 INFO:tasks.rados.rados.0.smithi026.stdout:3127: finishing rollback tid 0 to smithi02683976-32 2024-09-13T14:29:42.003 INFO:tasks.rados.rados.0.smithi026.stdout:3127: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3127: finishing rollback tid 2 to smithi02683976-32 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1498 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3128: finishing copy_from racing read to smithi02683976-1 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3128: finishing copy_from to smithi02683976-1 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1308 (ObjNum 931 snap 290 seq_num 931) dirty exists 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3127: done (2 left) 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3128: done (1 left) 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3129: done (0 left) 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3130: copy_from oid 20 from oid 32 current snap is 306 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3131: read oid 42 snap -1 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3131: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3132: read oid 19 snap -1 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:3132: expect (ObjNum 960 snap 300 seq_num 960) 2024-09-13T14:29:42.004 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 306 2024-09-13T14:29:42.005 INFO:tasks.rados.rados.0.smithi026.stdout:3133: seq_num 973 ranges {2367488=704512,3072000=573440,3645440=507904,4153344=122880} 2024-09-13T14:29:42.023 INFO:tasks.rados.rados.0.smithi026.stdout:3133: writing smithi02683976-24 from 2367488 to 3072000 tid 1 2024-09-13T14:29:42.028 INFO:tasks.rados.rados.0.smithi026.stdout:3133: writing smithi02683976-24 from 3072000 to 3645440 tid 2 2024-09-13T14:29:42.031 INFO:tasks.rados.rados.0.smithi026.stdout:3133: writing smithi02683976-24 from 3645440 to 4153344 tid 3 2024-09-13T14:29:42.032 INFO:tasks.rados.rados.0.smithi026.stdout:3133: writing smithi02683976-24 from 4153344 to 4276224 tid 4 2024-09-13T14:29:42.033 INFO:tasks.rados.rados.0.smithi026.stdout:3134: rmattr oid 21 current snap is 306 2024-09-13T14:29:42.035 INFO:tasks.rados.rados.0.smithi026.stdout:3131: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-13T14:29:42.141 INFO:tasks.rados.rados.0.smithi026.stdout:3131: done (4 left) 2024-09-13T14:29:42.141 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 306 2024-09-13T14:29:42.141 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 1 to smithi02683976-24 2024-09-13T14:29:42.141 INFO:tasks.rados.rados.0.smithi026.stdout:3132: expect (ObjNum 960 snap 300 seq_num 960) 2024-09-13T14:29:42.302 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1615 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:29:42.302 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 2 to smithi02683976-24 2024-09-13T14:29:42.302 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 3 to smithi02683976-24 2024-09-13T14:29:42.302 INFO:tasks.rados.rados.0.smithi026.stdout:3135: seq_num 974 ranges {0=720896,720896=507904,1228800=671744,1900544=24576} 2024-09-13T14:29:42.306 INFO:tasks.rados.rados.0.smithi026.stdout:3135: writing smithi02683976-48 from 0 to 720896 tid 1 2024-09-13T14:29:42.309 INFO:tasks.rados.rados.0.smithi026.stdout:3135: writing smithi02683976-48 from 720896 to 1228800 tid 2 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3135: writing smithi02683976-48 from 1228800 to 1900544 tid 3 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3135: writing smithi02683976-48 from 1900544 to 1925120 tid 4 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3132: done (4 left) 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3134: done (3 left) 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3136: read oid 46 snap -1 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3136: expect (ObjNum 964 snap 302 seq_num 964) 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3137: copy_from oid 8 from oid 43 current snap is 306 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 4 to smithi02683976-24 2024-09-13T14:29:42.314 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 5 to smithi02683976-24 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3133: finishing write tid 6 to smithi02683976-24 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1285 (ObjNum 973 snap 306 seq_num 973) dirty exists 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3133: left oid 24 (ObjNum 973 snap 306 seq_num 973) 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3130: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3130: finishing copy_from to smithi02683976-20 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1498 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 1 to smithi02683976-48 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 2 to smithi02683976-48 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3130: done (4 left) 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3133: done (3 left) 2024-09-13T14:29:42.315 INFO:tasks.rados.rados.0.smithi026.stdout:3138: snap_remove snap 299 2024-09-13T14:29:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[99461]: pgmap v615: 121 pgs: 121 active+clean; 572 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-13T14:29:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[99461]: osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:29:42.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[97022]: pgmap v615: 121 pgs: 121 active+clean; 572 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-13T14:29:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[97022]: osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:29:42.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:42.416 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:29:42 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:29:42.032+0000 7f85df2a1640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 18a after trimming was started 2024-09-13T14:29:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:42 smithi145 ceph-mon[89145]: pgmap v615: 121 pgs: 121 active+clean; 572 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-13T14:29:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:42 smithi145 ceph-mon[89145]: osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:29:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:43.015 INFO:tasks.rados.rados.0.smithi026.stdout:3137: finishing copy_from to smithi02683976-8 2024-09-13T14:29:43.015 INFO:tasks.rados.rados.0.smithi026.stdout:3137: got expected ENOENT (src dne) 2024-09-13T14:29:43.015 INFO:tasks.rados.rados.0.smithi026.stdout:3137: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:29:43.015 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 3 to smithi02683976-48 2024-09-13T14:29:43.015 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 4 to smithi02683976-48 2024-09-13T14:29:43.016 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 5 to smithi02683976-48 2024-09-13T14:29:43.016 INFO:tasks.rados.rados.0.smithi026.stdout:3135: finishing write tid 6 to smithi02683976-48 2024-09-13T14:29:43.016 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1166 (ObjNum 974 snap 306 seq_num 974) dirty exists 2024-09-13T14:29:43.016 INFO:tasks.rados.rados.0.smithi026.stdout:3135: left oid 48 (ObjNum 974 snap 306 seq_num 974) 2024-09-13T14:29:43.016 INFO:tasks.rados.rados.0.smithi026.stdout:3136: expect (ObjNum 964 snap 302 seq_num 964) 2024-09-13T14:29:43.315 INFO:tasks.rados.rados.0.smithi026.stdout:3135: done (3 left) 2024-09-13T14:29:43.316 INFO:tasks.rados.rados.0.smithi026.stdout:3136: done (2 left) 2024-09-13T14:29:43.316 INFO:tasks.rados.rados.0.smithi026.stdout:3137: done (1 left) 2024-09-13T14:29:43.316 INFO:tasks.rados.rados.0.smithi026.stdout:3138: done (0 left) 2024-09-13T14:29:43.316 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 306 2024-09-13T14:29:43.316 INFO:tasks.rados.rados.0.smithi026.stdout:3139: seq_num 975 ranges {376832=737280,1114112=786432,1900544=417792} 2024-09-13T14:29:43.321 INFO:tasks.rados.rados.0.smithi026.stdout:3139: writing smithi02683976-41 from 376832 to 1114112 tid 1 2024-09-13T14:29:43.326 INFO:tasks.rados.rados.0.smithi026.stdout:3139: writing smithi02683976-41 from 1114112 to 1900544 tid 2 2024-09-13T14:29:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:3139: writing smithi02683976-41 from 1900544 to 2318336 tid 3 2024-09-13T14:29:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 306 2024-09-13T14:29:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:3139: finishing write tid 1 to smithi02683976-41 2024-09-13T14:29:43.328 INFO:tasks.rados.rados.0.smithi026.stdout:3140: seq_num 976 ranges {2367488=753664,3121152=671744,3792896=532480} 2024-09-13T14:29:43.345 INFO:tasks.rados.rados.0.smithi026.stdout:3140: writing smithi02683976-2 from 2367488 to 3121152 tid 1 2024-09-13T14:29:43.349 INFO:tasks.rados.rados.0.smithi026.stdout:3140: writing smithi02683976-2 from 3121152 to 3792896 tid 2 2024-09-13T14:29:43.352 INFO:tasks.rados.rados.0.smithi026.stdout:3140: writing smithi02683976-2 from 3792896 to 4325376 tid 3 2024-09-13T14:29:43.352 INFO:tasks.rados.rados.0.smithi026.stdout:3141: setattr oid 18 current snap is 306 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: finishing write tid 2 to smithi02683976-41 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: finishing write tid 3 to smithi02683976-41 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: finishing write tid 4 to smithi02683976-41 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: finishing write tid 5 to smithi02683976-41 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1210 (ObjNum 975 snap 306 seq_num 975) dirty exists 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: left oid 41 (ObjNum 975 snap 306 seq_num 975) 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3140: finishing write tid 1 to smithi02683976-2 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3140: finishing write tid 2 to smithi02683976-2 2024-09-13T14:29:43.353 INFO:tasks.rados.rados.0.smithi026.stdout:3139: done (2 left) 2024-09-13T14:29:43.354 INFO:tasks.rados.rados.0.smithi026.stdout:3142: snap_remove snap 293 2024-09-13T14:29:43.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:43 smithi026 ceph-mon[97022]: osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:29:43.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:43 smithi026 ceph-mon[99461]: osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:29:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:43 smithi145 ceph-mon[89145]: osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:29:44.000 INFO:tasks.rados.rados.0.smithi026.stdout:3142: done (2 left) 2024-09-13T14:29:44.000 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 306 2024-09-13T14:29:44.000 INFO:tasks.rados.rados.0.smithi026.stdout:3143: seq_num 977 ranges {5259264=770048,6029312=557056,6586368=491520,7077888=155648} 2024-09-13T14:29:44.034 INFO:tasks.rados.rados.0.smithi026.stdout:3143: writing smithi02683976-38 from 5259264 to 6029312 tid 1 2024-09-13T14:29:44.038 INFO:tasks.rados.rados.0.smithi026.stdout:3143: writing smithi02683976-38 from 6029312 to 6586368 tid 2 2024-09-13T14:29:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:3143: writing smithi02683976-38 from 6586368 to 7077888 tid 3 2024-09-13T14:29:44.042 INFO:tasks.rados.rados.0.smithi026.stdout:3143: writing smithi02683976-38 from 7077888 to 7233536 tid 4 2024-09-13T14:29:44.042 INFO:tasks.rados.rados.0.smithi026.stdout:3144: read oid 10 snap -1 2024-09-13T14:29:44.042 INFO:tasks.rados.rados.0.smithi026.stdout:3144: expect (ObjNum 835 snap 260 seq_num 835) 2024-09-13T14:29:44.042 INFO:tasks.rados.rados.0.smithi026.stdout:3145: read oid 24 snap -1 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3145: expect (ObjNum 973 snap 306 seq_num 973) 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3146: snap_create 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3140: finishing write tid 3 to smithi02683976-2 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3140: finishing write tid 4 to smithi02683976-2 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1515 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3140: finishing write tid 5 to smithi02683976-2 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1289 (ObjNum 976 snap 306 seq_num 976) dirty exists 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3140: left oid 2 (ObjNum 976 snap 306 seq_num 976) 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 1 to smithi02683976-38 2024-09-13T14:29:44.043 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 2 to smithi02683976-38 2024-09-13T14:29:44.050 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 3 to smithi02683976-38 2024-09-13T14:29:44.051 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 4 to smithi02683976-38 2024-09-13T14:29:44.051 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 5 to smithi02683976-38 2024-09-13T14:29:44.051 INFO:tasks.rados.rados.0.smithi026.stdout:3143: finishing write tid 6 to smithi02683976-38 2024-09-13T14:29:44.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1529 (ObjNum 977 snap 306 seq_num 977) dirty exists 2024-09-13T14:29:44.051 INFO:tasks.rados.rados.0.smithi026.stdout:3143: left oid 38 (ObjNum 977 snap 306 seq_num 977) 2024-09-13T14:29:44.059 INFO:tasks.rados.rados.0.smithi026.stdout:3144: expect (ObjNum 835 snap 260 seq_num 835) 2024-09-13T14:29:44.175 INFO:tasks.rados.rados.0.smithi026.stdout:3145: expect (ObjNum 973 snap 306 seq_num 973) 2024-09-13T14:29:44.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[99461]: pgmap v618: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 17 op/s 2024-09-13T14:29:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[99461]: osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:29:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[99461]: osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:29:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[97022]: pgmap v618: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 17 op/s 2024-09-13T14:29:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[97022]: osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:29:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:44 smithi026 ceph-mon[97022]: osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:29:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:44 smithi145 ceph-mon[89145]: pgmap v618: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 17 op/s 2024-09-13T14:29:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:44 smithi145 ceph-mon[89145]: osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:29:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:44 smithi145 ceph-mon[89145]: osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:29:45.022 INFO:tasks.rados.rados.0.smithi026.stdout:3140: done (5 left) 2024-09-13T14:29:45.022 INFO:tasks.rados.rados.0.smithi026.stdout:3141: done (4 left) 2024-09-13T14:29:45.022 INFO:tasks.rados.rados.0.smithi026.stdout:3143: done (3 left) 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3144: done (2 left) 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3145: done (1 left) 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3146: done (0 left) 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3147: copy_from oid 49 from oid 13 current snap is 307 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3148: copy_from oid 46 from oid 17 current snap is 307 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 307 2024-09-13T14:29:45.023 INFO:tasks.rados.rados.0.smithi026.stdout:3149: seq_num 978 ranges {0=786432,786432=442368,1228800=647168,1875968=114688} 2024-09-13T14:29:45.027 INFO:tasks.rados.rados.0.smithi026.stdout:3149: writing smithi02683976-25 from 0 to 786432 tid 1 2024-09-13T14:29:45.029 INFO:tasks.rados.rados.0.smithi026.stdout:3149: writing smithi02683976-25 from 786432 to 1228800 tid 2 2024-09-13T14:29:45.034 INFO:tasks.rados.rados.0.smithi026.stdout:3149: writing smithi02683976-25 from 1228800 to 1875968 tid 3 2024-09-13T14:29:45.036 INFO:tasks.rados.rados.0.smithi026.stdout:3149: writing smithi02683976-25 from 1875968 to 1990656 tid 4 2024-09-13T14:29:45.036 INFO:tasks.rados.rados.0.smithi026.stdout:3150: read oid 11 snap -1 2024-09-13T14:29:45.036 INFO:tasks.rados.rados.0.smithi026.stdout:3150: expect (ObjNum 971 snap 303 seq_num 971) 2024-09-13T14:29:45.036 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 307 2024-09-13T14:29:45.036 INFO:tasks.rados.rados.0.smithi026.stdout:3151: seq_num 979 ranges {3473408=802816,4276224=729088,5005312=458752,5464064=16384} 2024-09-13T14:29:45.060 INFO:tasks.rados.rados.0.smithi026.stdout:3151: writing smithi02683976-12 from 3473408 to 4276224 tid 1 2024-09-13T14:29:45.064 INFO:tasks.rados.rados.0.smithi026.stdout:3151: writing smithi02683976-12 from 4276224 to 5005312 tid 2 2024-09-13T14:29:45.067 INFO:tasks.rados.rados.0.smithi026.stdout:3151: writing smithi02683976-12 from 5005312 to 5464064 tid 3 2024-09-13T14:29:45.067 INFO:tasks.rados.rados.0.smithi026.stdout:3151: writing smithi02683976-12 from 5464064 to 5480448 tid 4 2024-09-13T14:29:45.067 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 307 2024-09-13T14:29:45.067 INFO:tasks.rados.rados.0.smithi026.stdout:3147: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:29:45.067 INFO:tasks.rados.rados.0.smithi026.stdout:3152: seq_num 980 ranges {12083200=417792,12500992=614400,13115392=663552,13778944=327680} 2024-09-13T14:29:45.135 INFO:tasks.rados.rados.0.smithi026.stdout:3152: writing smithi02683976-32 from 12083200 to 12500992 tid 1 2024-09-13T14:29:45.139 INFO:tasks.rados.rados.0.smithi026.stdout:3152: writing smithi02683976-32 from 12500992 to 13115392 tid 2 2024-09-13T14:29:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3152: writing smithi02683976-32 from 13115392 to 13778944 tid 3 2024-09-13T14:29:45.145 INFO:tasks.rados.rados.0.smithi026.stdout:3152: writing smithi02683976-32 from 13778944 to 14106624 tid 4 2024-09-13T14:29:45.145 INFO:tasks.rados.rados.0.smithi026.stdout:3153: rollback oid 41 current snap is 307 2024-09-13T14:29:45.145 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 41 to 290 2024-09-13T14:29:45.145 INFO:tasks.rados.rados.0.smithi026.stdout:3147: finishing copy_from to smithi02683976-49 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1610 (ObjNum 948 snap 296 seq_num 948) dirty exists 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 1 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 2 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 3 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 4 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 5 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: finishing write tid 6 to smithi02683976-25 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1294 (ObjNum 978 snap 307 seq_num 978) dirty exists 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3149: left oid 25 (ObjNum 978 snap 307 seq_num 978) 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3148: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3148: finishing copy_from to smithi02683976-46 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1206 (ObjNum 959 snap 300 seq_num 959) dirty exists 2024-09-13T14:29:45.146 INFO:tasks.rados.rados.0.smithi026.stdout:3150: expect (ObjNum 971 snap 303 seq_num 971) 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 1 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 2 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 3 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 4 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 5 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: finishing write tid 6 to smithi02683976-12 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1453 (ObjNum 979 snap 307 seq_num 979) dirty exists 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3151: left oid 12 (ObjNum 979 snap 307 seq_num 979) 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 1 to smithi02683976-32 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 2 to smithi02683976-32 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 3 to smithi02683976-32 2024-09-13T14:29:45.260 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 4 to smithi02683976-32 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3153: finishing rollback tid 0 to smithi02683976-41 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 5 to smithi02683976-32 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3153: finishing rollback tid 1 to smithi02683976-41 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3153: finishing rollback tid 2 to smithi02683976-41 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1213 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3152: finishing write tid 6 to smithi02683976-32 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1535 (ObjNum 980 snap 307 seq_num 980) dirty exists 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3152: left oid 32 (ObjNum 980 snap 307 seq_num 980) 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3147: done (6 left) 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3148: done (5 left) 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3149: done (4 left) 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3150: done (3 left) 2024-09-13T14:29:45.261 INFO:tasks.rados.rados.0.smithi026.stdout:3151: done (2 left) 2024-09-13T14:29:45.262 INFO:tasks.rados.rados.0.smithi026.stdout:3152: done (1 left) 2024-09-13T14:29:45.262 INFO:tasks.rados.rados.0.smithi026.stdout:3153: done (0 left) 2024-09-13T14:29:45.262 INFO:tasks.rados.rados.0.smithi026.stdout:3154: delete oid 35 current snap is 307 2024-09-13T14:29:45.262 INFO:tasks.rados.rados.0.smithi026.stdout:3154: done (0 left) 2024-09-13T14:29:45.263 INFO:tasks.rados.rados.0.smithi026.stdout:3155: read oid 44 snap -1 2024-09-13T14:29:45.263 INFO:tasks.rados.rados.0.smithi026.stdout:3155: expect (ObjNum 949 snap 296 seq_num 949) 2024-09-13T14:29:45.263 INFO:tasks.rados.rados.0.smithi026.stdout:3156: read oid 36 snap 289 2024-09-13T14:29:45.263 INFO:tasks.rados.rados.0.smithi026.stdout:3156: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:29:45.263 INFO:tasks.rados.rados.0.smithi026.stdout:3157: snap_remove snap 288 2024-09-13T14:29:46.027 INFO:tasks.rados.rados.0.smithi026.stdout:3157: done (2 left) 2024-09-13T14:29:46.027 INFO:tasks.rados.rados.0.smithi026.stdout:3158: snap_remove snap 305 2024-09-13T14:29:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:46 smithi026 ceph-mon[97022]: pgmap v621: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 928 KiB/s wr, 3 op/s 2024-09-13T14:29:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:46 smithi026 ceph-mon[97022]: osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:29:46.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:46 smithi026 ceph-mon[99461]: pgmap v621: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 928 KiB/s wr, 3 op/s 2024-09-13T14:29:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:46 smithi026 ceph-mon[99461]: osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:29:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:46 smithi145 ceph-mon[89145]: pgmap v621: 121 pgs: 121 active+clean; 576 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 928 KiB/s wr, 3 op/s 2024-09-13T14:29:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:46 smithi145 ceph-mon[89145]: osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:29:47.034 INFO:tasks.rados.rados.0.smithi026.stdout:3158: done (2 left) 2024-09-13T14:29:47.035 INFO:tasks.rados.rados.0.smithi026.stdout:3159: read oid 33 snap 304 2024-09-13T14:29:47.035 INFO:tasks.rados.rados.0.smithi026.stdout:3159: expect deleted 2024-09-13T14:29:47.035 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 307 2024-09-13T14:29:47.035 INFO:tasks.rados.rados.0.smithi026.stdout:3160: seq_num 982 ranges {1736704=450560,2187264=425984,2613248=679936,3293184=466944,3760128=32768} 2024-09-13T14:29:47.047 INFO:tasks.rados.rados.0.smithi026.stdout:3160: writing smithi02683976-31 from 1736704 to 2187264 tid 1 2024-09-13T14:29:47.049 INFO:tasks.rados.rados.0.smithi026.stdout:3160: writing smithi02683976-31 from 2187264 to 2613248 tid 2 2024-09-13T14:29:47.053 INFO:tasks.rados.rados.0.smithi026.stdout:3160: writing smithi02683976-31 from 2613248 to 3293184 tid 3 2024-09-13T14:29:47.056 INFO:tasks.rados.rados.0.smithi026.stdout:3160: writing smithi02683976-31 from 3293184 to 3760128 tid 4 2024-09-13T14:29:47.056 INFO:tasks.rados.rados.0.smithi026.stdout:3160: writing smithi02683976-31 from 3760128 to 3792896 tid 5 2024-09-13T14:29:47.056 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 307 2024-09-13T14:29:47.056 INFO:tasks.rados.rados.0.smithi026.stdout:3161: seq_num 983 ranges {376832=466944,843776=712704,1556480=434176,1990656=458752} 2024-09-13T14:29:47.061 INFO:tasks.rados.rados.0.smithi026.stdout:3161: writing smithi02683976-41 from 376832 to 843776 tid 1 2024-09-13T14:29:47.065 INFO:tasks.rados.rados.0.smithi026.stdout:3161: writing smithi02683976-41 from 843776 to 1556480 tid 2 2024-09-13T14:29:47.067 INFO:tasks.rados.rados.0.smithi026.stdout:3161: writing smithi02683976-41 from 1556480 to 1990656 tid 3 2024-09-13T14:29:47.070 INFO:tasks.rados.rados.0.smithi026.stdout:3161: writing smithi02683976-41 from 1990656 to 2449408 tid 4 2024-09-13T14:29:47.070 INFO:tasks.rados.rados.0.smithi026.stdout:3162: delete oid 2 current snap is 307 2024-09-13T14:29:47.070 INFO:tasks.rados.rados.0.smithi026.stdout:3156: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:29:47.237 INFO:tasks.rados.rados.0.smithi026.stdout:3155: expect (ObjNum 949 snap 296 seq_num 949) 2024-09-13T14:29:47.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:47 smithi026 ceph-mon[99461]: osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:29:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:47 smithi026 ceph-mon[97022]: osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:29:47.431 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 1 to smithi02683976-31 2024-09-13T14:29:47.431 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 2 to smithi02683976-31 2024-09-13T14:29:47.432 INFO:tasks.rados.rados.0.smithi026.stdout:3155: done (5 left) 2024-09-13T14:29:47.432 INFO:tasks.rados.rados.0.smithi026.stdout:3156: done (4 left) 2024-09-13T14:29:47.432 INFO:tasks.rados.rados.0.smithi026.stdout:3159: done (3 left) 2024-09-13T14:29:47.432 INFO:tasks.rados.rados.0.smithi026.stdout:3162: done (2 left) 2024-09-13T14:29:47.432 INFO:tasks.rados.rados.0.smithi026.stdout:3163: snap_remove snap 302 2024-09-13T14:29:47.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:47 smithi145 ceph-mon[89145]: osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:29:48.037 INFO:tasks.rados.rados.0.smithi026.stdout:3163: done (2 left) 2024-09-13T14:29:48.038 INFO:tasks.rados.rados.0.smithi026.stdout:3164: copy_from oid 6 from oid 14 current snap is 307 2024-09-13T14:29:48.038 INFO:tasks.rados.rados.0.smithi026.stdout:3165: snap_remove snap 289 2024-09-13T14:29:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:48 smithi026 ceph-mon[97022]: pgmap v624: 121 pgs: 121 active+clean; 594 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:48 smithi026 ceph-mon[97022]: osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:29:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:48 smithi026 ceph-mon[99461]: pgmap v624: 121 pgs: 121 active+clean; 594 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:48 smithi026 ceph-mon[99461]: osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:29:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:48 smithi145 ceph-mon[89145]: pgmap v624: 121 pgs: 121 active+clean; 594 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 5.2 MiB/s wr, 16 op/s 2024-09-13T14:29:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:48 smithi145 ceph-mon[89145]: osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3165: done (3 left) 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3166: read oid 49 snap -1 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3166: expect (ObjNum 948 snap 296 seq_num 948) 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3167: read oid 1 snap -1 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3167: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3168: delete oid 20 current snap is 307 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 3 to smithi02683976-31 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 4 to smithi02683976-31 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 5 to smithi02683976-31 2024-09-13T14:29:49.004 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 6 to smithi02683976-31 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3160: finishing write tid 7 to smithi02683976-31 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1621 (ObjNum 982 snap 307 seq_num 982) dirty exists 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3160: left oid 31 (ObjNum 982 snap 307 seq_num 982) 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 1 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 2 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 3 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 4 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 5 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: finishing write tid 6 to smithi02683976-41 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1218 (ObjNum 983 snap 307 seq_num 983) dirty exists 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3161: left oid 41 (ObjNum 983 snap 307 seq_num 983) 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3164: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:29:49.005 INFO:tasks.rados.rados.0.smithi026.stdout:3164: finishing copy_from to smithi02683976-6 2024-09-13T14:29:49.006 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1489 (ObjNum 968 snap 302 seq_num 968) dirty exists 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3160: done (5 left) 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3161: done (4 left) 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3164: done (3 left) 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3168: done (2 left) 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3169: read oid 6 snap 271 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3169: expect deleted 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:3170: copy_from oid 26 from oid 33 current snap is 307 2024-09-13T14:29:49.018 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 307 2024-09-13T14:29:49.019 INFO:tasks.rados.rados.0.smithi026.stdout:3171: seq_num 986 ranges {0=516096,516096=770048,1286144=655360,1941504=180224} 2024-09-13T14:29:49.021 INFO:tasks.rados.rados.0.smithi026.stdout:3171: writing smithi02683976-37 from 0 to 516096 tid 1 2024-09-13T14:29:49.026 INFO:tasks.rados.rados.0.smithi026.stdout:3171: writing smithi02683976-37 from 516096 to 1286144 tid 2 2024-09-13T14:29:49.031 INFO:tasks.rados.rados.0.smithi026.stdout:3171: writing smithi02683976-37 from 1286144 to 1941504 tid 3 2024-09-13T14:29:49.032 INFO:tasks.rados.rados.0.smithi026.stdout:3171: writing smithi02683976-37 from 1941504 to 2121728 tid 4 2024-09-13T14:29:49.034 INFO:tasks.rados.rados.0.smithi026.stdout:3166: expect (ObjNum 948 snap 296 seq_num 948) 2024-09-13T14:29:49.079 INFO:tasks.rados.rados.0.smithi026.stdout:3166: done (4 left) 2024-09-13T14:29:49.079 INFO:tasks.rados.rados.0.smithi026.stdout:3169: done (3 left) 2024-09-13T14:29:49.079 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 307 2024-09-13T14:29:49.079 INFO:tasks.rados.rados.0.smithi026.stdout:3172: seq_num 987 ranges {2670592=532480,3203072=647168,3850240=417792,4268032=409600,4677632=131072} 2024-09-13T14:29:49.096 INFO:tasks.rados.rados.0.smithi026.stdout:3172: writing smithi02683976-16 from 2670592 to 3203072 tid 1 2024-09-13T14:29:49.100 INFO:tasks.rados.rados.0.smithi026.stdout:3172: writing smithi02683976-16 from 3203072 to 3850240 tid 2 2024-09-13T14:29:49.102 INFO:tasks.rados.rados.0.smithi026.stdout:3172: writing smithi02683976-16 from 3850240 to 4268032 tid 3 2024-09-13T14:29:49.105 INFO:tasks.rados.rados.0.smithi026.stdout:3172: writing smithi02683976-16 from 4268032 to 4677632 tid 4 2024-09-13T14:29:49.106 INFO:tasks.rados.rados.0.smithi026.stdout:3172: writing smithi02683976-16 from 4677632 to 4808704 tid 5 2024-09-13T14:29:49.106 INFO:tasks.rados.rados.0.smithi026.stdout:3173: copy_from oid 28 from oid 23 current snap is 307 2024-09-13T14:29:49.106 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 1 to smithi02683976-37 2024-09-13T14:29:49.106 INFO:tasks.rados.rados.0.smithi026.stdout:3170: finishing copy_from to smithi02683976-26 2024-09-13T14:29:49.106 INFO:tasks.rados.rados.0.smithi026.stdout:3170: got expected ENOENT (src dne) 2024-09-13T14:29:49.107 INFO:tasks.rados.rados.0.smithi026.stdout:3170: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:29:49.107 INFO:tasks.rados.rados.0.smithi026.stdout:3167: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:49.274 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 2 to smithi02683976-37 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 3 to smithi02683976-37 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 4 to smithi02683976-37 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 5 to smithi02683976-37 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: finishing write tid 6 to smithi02683976-37 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1465 (ObjNum 986 snap 307 seq_num 986) dirty exists 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: left oid 37 (ObjNum 986 snap 307 seq_num 986) 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 1 to smithi02683976-16 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 2 to smithi02683976-16 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 3 to smithi02683976-16 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3167: done (4 left) 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3170: done (3 left) 2024-09-13T14:29:49.275 INFO:tasks.rados.rados.0.smithi026.stdout:3171: done (2 left) 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3174: snap_create 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 4 to smithi02683976-16 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 5 to smithi02683976-16 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 6 to smithi02683976-16 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3172: finishing write tid 7 to smithi02683976-16 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1624 (ObjNum 987 snap 307 seq_num 987) dirty exists 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3172: left oid 16 (ObjNum 987 snap 307 seq_num 987) 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3173: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:3173: finishing copy_from to smithi02683976-28 2024-09-13T14:29:49.276 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1369 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:29:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:49 smithi026 ceph-mon[97022]: osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:29:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:49 smithi026 ceph-mon[97022]: osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:29:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:49 smithi026 ceph-mon[99461]: osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:29:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:49 smithi026 ceph-mon[99461]: osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:29:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:49 smithi145 ceph-mon[89145]: osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:29:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:49 smithi145 ceph-mon[89145]: osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:29:50.054 INFO:tasks.rados.rados.0.smithi026.stdout:3172: done (2 left) 2024-09-13T14:29:50.055 INFO:tasks.rados.rados.0.smithi026.stdout:3173: done (1 left) 2024-09-13T14:29:50.055 INFO:tasks.rados.rados.0.smithi026.stdout:3174: done (0 left) 2024-09-13T14:29:50.055 INFO:tasks.rados.rados.0.smithi026.stdout:3175: read oid 38 snap 306 2024-09-13T14:29:50.055 INFO:tasks.rados.rados.0.smithi026.stdout:3175: expect (ObjNum 977 snap 306 seq_num 977) 2024-09-13T14:29:50.055 INFO:tasks.rados.rados.0.smithi026.stdout:3176: delete oid 20 current snap is 308 2024-09-13T14:29:50.060 INFO:tasks.rados.rados.0.smithi026.stdout:3176: done (1 left) 2024-09-13T14:29:50.060 INFO:tasks.rados.rados.0.smithi026.stdout:3177: snap_create 2024-09-13T14:29:50.080 INFO:tasks.rados.rados.0.smithi026.stdout:3175: expect (ObjNum 977 snap 306 seq_num 977) 2024-09-13T14:29:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:50 smithi026 ceph-mon[97022]: pgmap v627: 121 pgs: 121 active+clean; 598 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-13T14:29:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:50 smithi026 ceph-mon[99461]: pgmap v627: 121 pgs: 121 active+clean; 598 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-13T14:29:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:50 smithi145 ceph-mon[89145]: pgmap v627: 121 pgs: 121 active+clean; 598 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-13T14:29:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:29:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:29:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3175: done (1 left) 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3177: done (0 left) 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3178: rollback oid 31 current snap is 309 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 308 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3179: read oid 45 snap -1 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3179: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3180: read oid 1 snap -1 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3180: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3181: read oid 34 snap -1 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3181: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:51.061 INFO:tasks.rados.rados.0.smithi026.stdout:3182: copy_from oid 20 from oid 41 current snap is 309 2024-09-13T14:29:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:3183: read oid 23 snap -1 2024-09-13T14:29:51.062 INFO:tasks.rados.rados.0.smithi026.stdout:3183: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:29:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3184: read oid 43 snap 279 2024-09-13T14:29:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3184: expect deleted 2024-09-13T14:29:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3185: delete oid 11 current snap is 309 2024-09-13T14:29:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3178: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:29:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3180: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:29:51.289 INFO:tasks.rados.rados.0.smithi026.stdout:3178: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:29:51.289 INFO:tasks.rados.rados.0.smithi026.stdout:3178: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:29:51.289 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1624 (ObjNum 982 snap 307 seq_num 982) dirty exists 2024-09-13T14:29:51.289 INFO:tasks.rados.rados.0.smithi026.stdout:3178: done (7 left) 2024-09-13T14:29:51.289 INFO:tasks.rados.rados.0.smithi026.stdout:3180: done (6 left) 2024-09-13T14:29:51.290 INFO:tasks.rados.rados.0.smithi026.stdout:3185: done (5 left) 2024-09-13T14:29:51.290 INFO:tasks.rados.rados.0.smithi026.stdout:3186: snap_remove snap 290 2024-09-13T14:29:51.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:51 smithi026 ceph-mon[97022]: osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:29:51.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:51 smithi026 ceph-mon[99461]: osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:29:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:51 smithi145 ceph-mon[89145]: osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:29:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:29:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:29:51] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.51.0" 2024-09-13T14:29:52.065 INFO:tasks.rados.rados.0.smithi026.stdout:3186: done (5 left) 2024-09-13T14:29:52.065 INFO:tasks.rados.rados.0.smithi026.stdout:3187: read oid 29 snap -1 2024-09-13T14:29:52.065 INFO:tasks.rados.rados.0.smithi026.stdout:3187: expect deleted 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3188: rollback oid 7 current snap is 309 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 7 to 307 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3189: read oid 49 snap 235 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3189: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3190: read oid 10 snap 279 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3190: expect (ObjNum 835 snap 260 seq_num 835) 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3191: copy_from oid 21 from oid 18 current snap is 309 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3192: copy_from oid 4 from oid 27 current snap is 309 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 309 2024-09-13T14:29:52.066 INFO:tasks.rados.rados.0.smithi026.stdout:3193: seq_num 990 ranges {3842048=589824,4431872=753664,5185536=638976,5824512=212992} 2024-09-13T14:29:52.092 INFO:tasks.rados.rados.0.smithi026.stdout:3193: writing smithi02683976-9 from 3842048 to 4431872 tid 1 2024-09-13T14:29:52.097 INFO:tasks.rados.rados.0.smithi026.stdout:3193: writing smithi02683976-9 from 4431872 to 5185536 tid 2 2024-09-13T14:29:52.101 INFO:tasks.rados.rados.0.smithi026.stdout:3193: writing smithi02683976-9 from 5185536 to 5824512 tid 3 2024-09-13T14:29:52.103 INFO:tasks.rados.rados.0.smithi026.stdout:3193: writing smithi02683976-9 from 5824512 to 6037504 tid 4 2024-09-13T14:29:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3194: copy_from oid 26 from oid 11 current snap is 309 2024-09-13T14:29:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3195: snap_remove snap 263 2024-09-13T14:29:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:52 smithi026 ceph-mon[99461]: pgmap v630: 121 pgs: 121 active+clean; 625 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 11 MiB/s wr, 19 op/s 2024-09-13T14:29:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:52 smithi026 ceph-mon[99461]: osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:29:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:52 smithi026 ceph-mon[97022]: pgmap v630: 121 pgs: 121 active+clean; 625 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 11 MiB/s wr, 19 op/s 2024-09-13T14:29:52.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:52 smithi026 ceph-mon[97022]: osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:29:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:52 smithi145 ceph-mon[89145]: pgmap v630: 121 pgs: 121 active+clean; 625 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 11 MiB/s wr, 19 op/s 2024-09-13T14:29:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:52 smithi145 ceph-mon[89145]: osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:29:53.078 INFO:tasks.rados.rados.0.smithi026.stdout:3182: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:29:53.078 INFO:tasks.rados.rados.0.smithi026.stdout:3182: finishing copy_from to smithi02683976-20 2024-09-13T14:29:53.078 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 1218 (ObjNum 983 snap 307 seq_num 983) dirty exists 2024-09-13T14:29:53.078 INFO:tasks.rados.rados.0.smithi026.stdout:3183: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:29:53.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:53 smithi026 ceph-mon[99461]: osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:29:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:53 smithi026 ceph-mon[97022]: osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:29:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:53 smithi145 ceph-mon[89145]: osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:29:54.271 INFO:tasks.rados.rados.0.smithi026.stdout:3181: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:54.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[99461]: pgmap v633: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 17 op/s 2024-09-13T14:29:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[99461]: osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:29:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[99461]: osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:29:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[97022]: pgmap v633: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 17 op/s 2024-09-13T14:29:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[97022]: osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:29:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:54 smithi026 ceph-mon[97022]: osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:29:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:54 smithi145 ceph-mon[89145]: pgmap v633: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 17 op/s 2024-09-13T14:29:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:54 smithi145 ceph-mon[89145]: osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:29:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:54 smithi145 ceph-mon[89145]: osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:29:55.944 INFO:tasks.rados.rados.0.smithi026.stdout:3179: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:29:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:56 smithi026 ceph-mon[99461]: pgmap v636: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-13T14:29:56.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:56 smithi026 ceph-mon[97022]: pgmap v636: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-13T14:29:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:56 smithi145 ceph-mon[89145]: pgmap v636: 121 pgs: 121 active+clean; 627 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-13T14:29:57.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.623 INFO:tasks.rados.rados.0.smithi026.stdout:3188: finishing rollback tid 0 to smithi02683976-7 2024-09-13T14:29:57.623 INFO:tasks.rados.rados.0.smithi026.stdout:3188: finishing rollback tid 1 to smithi02683976-7 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3188: finishing rollback tid 2 to smithi02683976-7 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1602 (ObjNum 936 snap 292 seq_num 936) dirty exists 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3191: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3191: finishing copy_from to smithi02683976-21 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1515 (ObjNum 4065 snap 0 seq_num 3890193136) dirty exists 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3192: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3192: finishing copy_from to smithi02683976-4 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1815 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 1 to smithi02683976-9 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 2 to smithi02683976-9 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3194: finishing copy_from to smithi02683976-26 2024-09-13T14:29:57.624 INFO:tasks.rados.rados.0.smithi026.stdout:3194: got expected ENOENT (src dne) 2024-09-13T14:29:57.625 INFO:tasks.rados.rados.0.smithi026.stdout:3194: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:29:57.625 INFO:tasks.rados.rados.0.smithi026.stdout:3189: expect (ObjNum 703 snap 202 seq_num 703) 2024-09-13T14:29:57.897 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 3 to smithi02683976-9 2024-09-13T14:29:57.897 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 4 to smithi02683976-9 2024-09-13T14:29:57.897 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 5 to smithi02683976-9 2024-09-13T14:29:57.897 INFO:tasks.rados.rados.0.smithi026.stdout:3193: finishing write tid 6 to smithi02683976-9 2024-09-13T14:29:57.897 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1630 (ObjNum 990 snap 309 seq_num 990) dirty exists 2024-09-13T14:29:57.898 INFO:tasks.rados.rados.0.smithi026.stdout:3193: left oid 9 (ObjNum 990 snap 309 seq_num 990) 2024-09-13T14:29:57.898 INFO:tasks.rados.rados.0.smithi026.stdout:3190: expect (ObjNum 835 snap 260 seq_num 835) 2024-09-13T14:29:58.013 INFO:tasks.rados.rados.0.smithi026.stdout:3179: done (13 left) 2024-09-13T14:29:58.013 INFO:tasks.rados.rados.0.smithi026.stdout:3181: done (12 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3182: done (11 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3183: done (10 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3184: done (9 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3187: done (8 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3188: done (7 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3189: done (6 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3190: done (5 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3191: done (4 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3192: done (3 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3193: done (2 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3194: done (1 left) 2024-09-13T14:29:58.015 INFO:tasks.rados.rados.0.smithi026.stdout:3195: done (0 left) 2024-09-13T14:29:58.016 INFO:tasks.rados.rados.0.smithi026.stdout:3196: copy_from oid 35 from oid 17 current snap is 309 2024-09-13T14:29:58.016 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 309 2024-09-13T14:29:58.016 INFO:tasks.rados.rados.0.smithi026.stdout:3197: seq_num 991 ranges {1482752=606208,2088960=638976,2727936=794624,3522560=172032} 2024-09-13T14:29:58.025 INFO:tasks.rados.rados.0.smithi026.stdout:3197: writing smithi02683976-49 from 1482752 to 2088960 tid 1 2024-09-13T14:29:58.029 INFO:tasks.rados.rados.0.smithi026.stdout:3197: writing smithi02683976-49 from 2088960 to 2727936 tid 2 2024-09-13T14:29:58.034 INFO:tasks.rados.rados.0.smithi026.stdout:3197: writing smithi02683976-49 from 2727936 to 3522560 tid 3 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3197: writing smithi02683976-49 from 3522560 to 3694592 tid 4 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3198: rollback oid 36 current snap is 309 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 303 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3196: finishing copy_from racing read to smithi02683976-35 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3196: finishing copy_from to smithi02683976-35 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1206 (ObjNum 959 snap 300 seq_num 959) dirty exists 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 1 to smithi02683976-49 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 2 to smithi02683976-49 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3196: done (2 left) 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:3199: rollback oid 31 current snap is 309 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 303 2024-09-13T14:29:58.036 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 309 2024-09-13T14:29:58.037 INFO:tasks.rados.rados.0.smithi026.stdout:3200: seq_num 992 ranges {0=622592,622592=524288,1146880=606208,1753088=475136} 2024-09-13T14:29:58.040 INFO:tasks.rados.rados.0.smithi026.stdout:3200: writing smithi02683976-47 from 0 to 622592 tid 1 2024-09-13T14:29:58.043 INFO:tasks.rados.rados.0.smithi026.stdout:3200: writing smithi02683976-47 from 622592 to 1146880 tid 2 2024-09-13T14:29:58.047 INFO:tasks.rados.rados.0.smithi026.stdout:3200: writing smithi02683976-47 from 1146880 to 1753088 tid 3 2024-09-13T14:29:58.050 INFO:tasks.rados.rados.0.smithi026.stdout:3200: writing smithi02683976-47 from 1753088 to 2228224 tid 4 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3201: delete oid 25 current snap is 309 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 3 to smithi02683976-49 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 4 to smithi02683976-49 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3199: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 5 to smithi02683976-49 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3199: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3199: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1633 (ObjNum 963 snap 302 seq_num 963) dirty exists 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:3197: finishing write tid 6 to smithi02683976-49 2024-09-13T14:29:58.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1615 (ObjNum 991 snap 309 seq_num 991) dirty exists 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3197: left oid 49 (ObjNum 991 snap 309 seq_num 991) 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3198: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3198: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3198: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 1916 (ObjNum 864 snap 266 seq_num 864) dirty exists 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 1 to smithi02683976-47 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 2 to smithi02683976-47 2024-09-13T14:29:58.052 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 3 to smithi02683976-47 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3197: done (4 left) 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3198: done (3 left) 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3199: done (2 left) 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3201: done (1 left) 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3202: read oid 16 snap -1 2024-09-13T14:29:58.053 INFO:tasks.rados.rados.0.smithi026.stdout:3202: expect (ObjNum 987 snap 307 seq_num 987) 2024-09-13T14:29:58.054 INFO:tasks.rados.rados.0.smithi026.stdout:3203: delete oid 30 current snap is 309 2024-09-13T14:29:58.054 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 4 to smithi02683976-47 2024-09-13T14:29:58.054 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 5 to smithi02683976-47 2024-09-13T14:29:58.054 INFO:tasks.rados.rados.0.smithi026.stdout:3200: finishing write tid 6 to smithi02683976-47 2024-09-13T14:29:58.054 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1171 (ObjNum 992 snap 309 seq_num 992) dirty exists 2024-09-13T14:29:58.055 INFO:tasks.rados.rados.0.smithi026.stdout:3200: left oid 47 (ObjNum 992 snap 309 seq_num 992) 2024-09-13T14:29:58.055 INFO:tasks.rados.rados.0.smithi026.stdout:3200: done (2 left) 2024-09-13T14:29:58.055 INFO:tasks.rados.rados.0.smithi026.stdout:3203: done (1 left) 2024-09-13T14:29:58.055 INFO:tasks.rados.rados.0.smithi026.stdout:3204: copy_from oid 47 from oid 37 current snap is 309 2024-09-13T14:29:58.055 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 309 2024-09-13T14:29:58.056 INFO:tasks.rados.rados.0.smithi026.stdout:3205: seq_num 995 ranges {3923968=671744,4595712=573440,5169152=778240,5947392=253952} 2024-09-13T14:29:58.081 INFO:tasks.rados.rados.0.smithi026.stdout:3205: writing smithi02683976-1 from 3923968 to 4595712 tid 1 2024-09-13T14:29:58.085 INFO:tasks.rados.rados.0.smithi026.stdout:3205: writing smithi02683976-1 from 4595712 to 5169152 tid 2 2024-09-13T14:29:58.090 INFO:tasks.rados.rados.0.smithi026.stdout:3205: writing smithi02683976-1 from 5169152 to 5947392 tid 3 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3205: writing smithi02683976-1 from 5947392 to 6201344 tid 4 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3206: read oid 50 snap -1 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3206: expect deleted 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3207: delete oid 19 current snap is 309 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3204: finishing copy_from racing read to smithi02683976-47 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3204: finishing copy_from to smithi02683976-47 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1465 (ObjNum 986 snap 307 seq_num 986) dirty exists 2024-09-13T14:29:58.092 INFO:tasks.rados.rados.0.smithi026.stdout:3202: expect (ObjNum 987 snap 307 seq_num 987) 2024-09-13T14:29:58.294 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 1 to smithi02683976-1 2024-09-13T14:29:58.294 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 2 to smithi02683976-1 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 3 to smithi02683976-1 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 4 to smithi02683976-1 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 5 to smithi02683976-1 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: finishing write tid 6 to smithi02683976-1 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1922 (ObjNum 995 snap 309 seq_num 995) dirty exists 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: left oid 1 (ObjNum 995 snap 309 seq_num 995) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3202: done (4 left) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3204: done (3 left) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3205: done (2 left) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3206: done (1 left) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3207: done (0 left) 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 309 2024-09-13T14:29:58.295 INFO:tasks.rados.rados.0.smithi026.stdout:3208: seq_num 997 ranges {4276224=704512,4980736=794624,5775360=794624,6569984=16384} 2024-09-13T14:29:58.321 INFO:tasks.rados.rados.0.smithi026.stdout:3208: writing smithi02683976-24 from 4276224 to 4980736 tid 1 2024-09-13T14:29:58.326 INFO:tasks.rados.rados.0.smithi026.stdout:3208: writing smithi02683976-24 from 4980736 to 5775360 tid 2 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3208: writing smithi02683976-24 from 5775360 to 6569984 tid 3 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3208: writing smithi02683976-24 from 6569984 to 6586368 tid 4 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3209: rollback oid 13 current snap is 309 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 13 to 306 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 1 to smithi02683976-24 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 2 to smithi02683976-24 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 309 2024-09-13T14:29:58.331 INFO:tasks.rados.rados.0.smithi026.stdout:3210: seq_num 998 ranges {0=720896,720896=679936,1400832=598016,1998848=327680} 2024-09-13T14:29:58.335 INFO:tasks.rados.rados.0.smithi026.stdout:3210: writing smithi02683976-43 from 0 to 720896 tid 1 2024-09-13T14:29:58.340 INFO:tasks.rados.rados.0.smithi026.stdout:3210: writing smithi02683976-43 from 720896 to 1400832 tid 2 2024-09-13T14:29:58.343 INFO:tasks.rados.rados.0.smithi026.stdout:3210: writing smithi02683976-43 from 1400832 to 1998848 tid 3 2024-09-13T14:29:58.345 INFO:tasks.rados.rados.0.smithi026.stdout:3210: writing smithi02683976-43 from 1998848 to 2326528 tid 4 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 309 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 3 to smithi02683976-24 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 4 to smithi02683976-24 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:3209: finishing rollback tid 0 to smithi02683976-13 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:3209: finishing rollback tid 1 to smithi02683976-13 2024-09-13T14:29:58.346 INFO:tasks.rados.rados.0.smithi026.stdout:3211: seq_num 999 ranges {2367488=737280,3104768=565248,3670016=761856,4431872=278528} 2024-09-13T14:29:58.362 INFO:tasks.rados.rados.0.smithi026.stdout:3211: writing smithi02683976-40 from 2367488 to 3104768 tid 1 2024-09-13T14:29:58.365 INFO:tasks.rados.rados.0.smithi026.stdout:3211: writing smithi02683976-40 from 3104768 to 3670016 tid 2 2024-09-13T14:29:58.370 INFO:tasks.rados.rados.0.smithi026.stdout:3211: writing smithi02683976-40 from 3670016 to 4431872 tid 3 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3211: writing smithi02683976-40 from 4431872 to 4710400 tid 4 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3212: snap_create 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 5 to smithi02683976-24 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3209: finishing rollback tid 2 to smithi02683976-13 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 1628 (ObjNum 948 snap 296 seq_num 948) dirty exists 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3208: finishing write tid 6 to smithi02683976-24 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1301 (ObjNum 997 snap 309 seq_num 997) dirty exists 2024-09-13T14:29:58.372 INFO:tasks.rados.rados.0.smithi026.stdout:3208: left oid 24 (ObjNum 997 snap 309 seq_num 997) 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 1 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 2 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 3 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 4 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 5 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: finishing write tid 6 to smithi02683976-43 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1633 (ObjNum 998 snap 309 seq_num 998) dirty exists 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3210: left oid 43 (ObjNum 998 snap 309 seq_num 998) 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 1 to smithi02683976-40 2024-09-13T14:29:58.373 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 2 to smithi02683976-40 2024-09-13T14:29:58.374 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 3 to smithi02683976-40 2024-09-13T14:29:58.374 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 4 to smithi02683976-40 2024-09-13T14:29:58.377 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 5 to smithi02683976-40 2024-09-13T14:29:58.377 INFO:tasks.rados.rados.0.smithi026.stdout:3211: finishing write tid 6 to smithi02683976-40 2024-09-13T14:29:58.377 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1620 (ObjNum 999 snap 309 seq_num 999) dirty exists 2024-09-13T14:29:58.377 INFO:tasks.rados.rados.0.smithi026.stdout:3211: left oid 40 (ObjNum 999 snap 309 seq_num 999) 2024-09-13T14:29:58.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:58 smithi026 ceph-mon[97022]: pgmap v637: 121 pgs: 121 active+clean; 613 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-13T14:29:58.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:58 smithi026 ceph-mon[99461]: pgmap v637: 121 pgs: 121 active+clean; 613 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-13T14:29:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:58 smithi145 ceph-mon[89145]: pgmap v637: 121 pgs: 121 active+clean; 613 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-13T14:29:59.003 INFO:tasks.rados.rados.0.smithi026.stdout:3208: done (4 left) 2024-09-13T14:29:59.003 INFO:tasks.rados.rados.0.smithi026.stdout:3209: done (3 left) 2024-09-13T14:29:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:3210: done (2 left) 2024-09-13T14:29:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:3211: done (1 left) 2024-09-13T14:29:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:3212: done (0 left) 2024-09-13T14:29:59.004 INFO:tasks.rados.rados.0.smithi026.stdout:3213: delete oid 47 current snap is 310 2024-09-13T14:29:59.006 INFO:tasks.rados.rados.0.smithi026.stdout:3213: done (0 left) 2024-09-13T14:29:59.006 INFO:tasks.rados.rados.0.smithi026.stdout:3214: snap_create 2024-09-13T14:29:59.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:59 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:59.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:29:59 smithi026 ceph-mon[97022]: osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:29:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:59 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:29:59 smithi026 ceph-mon[99461]: osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:29:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:59 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:29:59 smithi145 ceph-mon[89145]: osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:3214: done (0 left) 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:3215: read oid 39 snap -1 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:3215: expect (ObjNum 972 snap 304 seq_num 972) 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:3216: read oid 43 snap -1 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:3216: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-13T14:30:00.011 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 311 2024-09-13T14:30:00.012 INFO:tasks.rados.rados.0.smithi026.stdout:3217: seq_num 1001 ranges {12083200=770048,12853248=729088,13582336=770048,14352384=106496} 2024-09-13T14:30:00.081 INFO:tasks.rados.rados.0.smithi026.stdout:3217: writing smithi02683976-26 from 12083200 to 12853248 tid 1 2024-09-13T14:30:00.085 INFO:tasks.rados.rados.0.smithi026.stdout:3217: writing smithi02683976-26 from 12853248 to 13582336 tid 2 2024-09-13T14:30:00.089 INFO:tasks.rados.rados.0.smithi026.stdout:3217: writing smithi02683976-26 from 13582336 to 14352384 tid 3 2024-09-13T14:30:00.090 INFO:tasks.rados.rados.0.smithi026.stdout:3217: writing smithi02683976-26 from 14352384 to 14458880 tid 4 2024-09-13T14:30:00.090 INFO:tasks.rados.rados.0.smithi026.stdout:3218: delete oid 4 current snap is 311 2024-09-13T14:30:00.090 INFO:tasks.rados.rados.0.smithi026.stdout:3216: expect (ObjNum 998 snap 309 seq_num 998) 2024-09-13T14:30:00.152 INFO:tasks.rados.rados.0.smithi026.stdout:3215: expect (ObjNum 972 snap 304 seq_num 972) 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 1 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 2 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 3 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 4 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 5 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: finishing write tid 6 to smithi02683976-26 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1458 (ObjNum 1001 snap 311 seq_num 1001) dirty exists 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3217: left oid 26 (ObjNum 1001 snap 311 seq_num 1001) 2024-09-13T14:30:00.250 INFO:tasks.rados.rados.0.smithi026.stdout:3215: done (3 left) 2024-09-13T14:30:00.251 INFO:tasks.rados.rados.0.smithi026.stdout:3216: done (2 left) 2024-09-13T14:30:00.251 INFO:tasks.rados.rados.0.smithi026.stdout:3217: done (1 left) 2024-09-13T14:30:00.251 INFO:tasks.rados.rados.0.smithi026.stdout:3218: done (0 left) 2024-09-13T14:30:00.251 INFO:tasks.rados.rados.0.smithi026.stdout:3219: setattr oid 33 current snap is 311 2024-09-13T14:30:00.251 INFO:tasks.rados.rados.0.smithi026.stdout:3220: delete oid 45 current snap is 311 2024-09-13T14:30:00.253 INFO:tasks.rados.rados.0.smithi026.stdout:3220: done (1 left) 2024-09-13T14:30:00.254 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 311 2024-09-13T14:30:00.254 INFO:tasks.rados.rados.0.smithi026.stdout:3221: seq_num 1004 ranges {2326528=16384} 2024-09-13T14:30:00.266 INFO:tasks.rados.rados.0.smithi026.stdout:3221: writing smithi02683976-43 from 2326528 to 2342912 tid 1 2024-09-13T14:30:00.266 INFO:tasks.rados.rados.0.smithi026.stdout:3222: delete oid 14 current snap is 311 2024-09-13T14:30:00.266 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1923 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:30:00.269 INFO:tasks.rados.rados.0.smithi026.stdout:3219: done (2 left) 2024-09-13T14:30:00.269 INFO:tasks.rados.rados.0.smithi026.stdout:3222: done (1 left) 2024-09-13T14:30:00.269 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 311 2024-09-13T14:30:00.269 INFO:tasks.rados.rados.0.smithi026.stdout:3223: seq_num 1006 ranges {2891776=49152} 2024-09-13T14:30:00.285 INFO:tasks.rados.rados.0.smithi026.stdout:3223: writing smithi02683976-5 from 2891776 to 2940928 tid 1 2024-09-13T14:30:00.285 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 311 2024-09-13T14:30:00.285 INFO:tasks.rados.rados.0.smithi026.stdout:3221: finishing write tid 1 to smithi02683976-43 2024-09-13T14:30:00.286 INFO:tasks.rados.rados.0.smithi026.stdout:3221: finishing write tid 2 to smithi02683976-43 2024-09-13T14:30:00.286 INFO:tasks.rados.rados.0.smithi026.stdout:3221: finishing write tid 3 to smithi02683976-43 2024-09-13T14:30:00.286 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1635 (ObjNum 1004 snap 311 seq_num 1004) dirty exists 2024-09-13T14:30:00.286 INFO:tasks.rados.rados.0.smithi026.stdout:3221: left oid 43 (ObjNum 1004 snap 311 seq_num 1004) 2024-09-13T14:30:00.286 INFO:tasks.rados.rados.0.smithi026.stdout:3224: seq_num 1007 ranges {1925120=65536} 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3224: writing smithi02683976-48 from 1925120 to 1990656 tid 1 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3221: done (2 left) 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3225: rollback oid 25 current snap is 311 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 298 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3223: finishing write tid 1 to smithi02683976-5 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3223: finishing write tid 2 to smithi02683976-5 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:3223: finishing write tid 3 to smithi02683976-5 2024-09-13T14:30:00.297 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1926 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:30:00.298 INFO:tasks.rados.rados.0.smithi026.stdout:3223: left oid 5 (ObjNum 1006 snap 311 seq_num 1006) 2024-09-13T14:30:00.298 INFO:tasks.rados.rados.0.smithi026.stdout:3223: done (2 left) 2024-09-13T14:30:00.298 INFO:tasks.rados.rados.0.smithi026.stdout:3226: delete oid 32 current snap is 311 2024-09-13T14:30:00.298 INFO:tasks.rados.rados.0.smithi026.stdout:3225: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:30:00.298 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1296 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-13T14:30:00.299 INFO:tasks.rados.rados.0.smithi026.stdout:3224: finishing write tid 1 to smithi02683976-48 2024-09-13T14:30:00.299 INFO:tasks.rados.rados.0.smithi026.stdout:3224: finishing write tid 2 to smithi02683976-48 2024-09-13T14:30:00.300 INFO:tasks.rados.rados.0.smithi026.stdout:3224: finishing write tid 3 to smithi02683976-48 2024-09-13T14:30:00.300 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1468 (ObjNum 1007 snap 311 seq_num 1007) dirty exists 2024-09-13T14:30:00.300 INFO:tasks.rados.rados.0.smithi026.stdout:3224: left oid 48 (ObjNum 1007 snap 311 seq_num 1007) 2024-09-13T14:30:00.300 INFO:tasks.rados.rados.0.smithi026.stdout:3224: done (2 left) 2024-09-13T14:30:00.300 INFO:tasks.rados.rados.0.smithi026.stdout:3225: done (1 left) 2024-09-13T14:30:00.301 INFO:tasks.rados.rados.0.smithi026.stdout:3226: done (0 left) 2024-09-13T14:30:00.301 INFO:tasks.rados.rados.0.smithi026.stdout:3227: delete oid 9 current snap is 311 2024-09-13T14:30:00.303 INFO:tasks.rados.rados.0.smithi026.stdout:3227: done (0 left) 2024-09-13T14:30:00.303 INFO:tasks.rados.rados.0.smithi026.stdout:3228: read oid 36 snap 285 2024-09-13T14:30:00.303 INFO:tasks.rados.rados.0.smithi026.stdout:3228: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:00.303 INFO:tasks.rados.rados.0.smithi026.stdout:3229: snap_remove snap 294 2024-09-13T14:30:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: pgmap v638: 121 pgs: 121 active+clean; 612 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: overall HEALTH_OK 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[97022]: osdmap e690: 8 total, 8 up, 8 in 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: pgmap v638: 121 pgs: 121 active+clean; 612 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:30:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: overall HEALTH_OK 2024-09-13T14:30:00.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:00 smithi026 ceph-mon[99461]: osdmap e690: 8 total, 8 up, 8 in 2024-09-13T14:30:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:00.456 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=infra.usagestats t=2024-09-13T14:30:00.178665717Z level=info msg="Usage stats are ready to report" 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: pgmap v638: 121 pgs: 121 active+clean; 612 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: overall HEALTH_OK 2024-09-13T14:30:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:00 smithi145 ceph-mon[89145]: osdmap e690: 8 total, 8 up, 8 in 2024-09-13T14:30:01.097 INFO:tasks.rados.rados.0.smithi026.stdout:3229: done (1 left) 2024-09-13T14:30:01.097 INFO:tasks.rados.rados.0.smithi026.stdout:3230: delete oid 44 current snap is 311 2024-09-13T14:30:01.098 INFO:tasks.rados.rados.0.smithi026.stdout:3228: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:01.274 INFO:tasks.rados.rados.0.smithi026.stdout:3228: done (1 left) 2024-09-13T14:30:01.275 INFO:tasks.rados.rados.0.smithi026.stdout:3230: done (0 left) 2024-09-13T14:30:01.275 INFO:tasks.rados.rados.0.smithi026.stdout:3231: delete oid 22 current snap is 311 2024-09-13T14:30:01.278 INFO:tasks.rados.rados.0.smithi026.stdout:3231: done (0 left) 2024-09-13T14:30:01.278 INFO:tasks.rados.rados.0.smithi026.stdout:3232: rollback oid 41 current snap is 311 2024-09-13T14:30:01.278 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 41 to 306 2024-09-13T14:30:01.278 INFO:tasks.rados.rados.0.smithi026.stdout:3233: rollback oid 19 current snap is 311 2024-09-13T14:30:01.278 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 19 to 285 2024-09-13T14:30:01.279 INFO:tasks.rados.rados.0.smithi026.stdout:3234: rollback oid 24 current snap is 311 2024-09-13T14:30:01.279 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 291 2024-09-13T14:30:01.279 INFO:tasks.rados.rados.0.smithi026.stdout:3235: snap_create 2024-09-13T14:30:01.282 INFO:tasks.rados.rados.0.smithi026.stdout:3232: finishing rollback tid 0 to smithi02683976-41 2024-09-13T14:30:01.282 INFO:tasks.rados.rados.0.smithi026.stdout:3232: finishing rollback tid 1 to smithi02683976-41 2024-09-13T14:30:01.282 INFO:tasks.rados.rados.0.smithi026.stdout:3233: finishing rollback tid 1 to smithi02683976-19 2024-09-13T14:30:01.283 INFO:tasks.rados.rados.0.smithi026.stdout:3232: finishing rollback tid 2 to smithi02683976-41 2024-09-13T14:30:01.283 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1221 (ObjNum 975 snap 306 seq_num 975) dirty exists 2024-09-13T14:30:01.283 INFO:tasks.rados.rados.0.smithi026.stdout:3234: finishing rollback tid 0 to smithi02683976-24 2024-09-13T14:30:01.283 INFO:tasks.rados.rados.0.smithi026.stdout:3233: finishing rollback tid 2 to smithi02683976-19 2024-09-13T14:30:01.283 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1637 (ObjNum 908 snap 282 seq_num 908) dirty exists 2024-09-13T14:30:01.284 INFO:tasks.rados.rados.0.smithi026.stdout:3234: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:30:01.284 INFO:tasks.rados.rados.0.smithi026.stdout:3234: finishing rollback tid 2 to smithi02683976-24 2024-09-13T14:30:01.284 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1304 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:30:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:01] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.51.0" 2024-09-13T14:30:02.101 INFO:tasks.rados.rados.0.smithi026.stdout:3232: done (3 left) 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3233: done (2 left) 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3234: done (1 left) 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3235: done (0 left) 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3236: read oid 49 snap -1 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3236: expect (ObjNum 991 snap 309 seq_num 991) 2024-09-13T14:30:02.102 INFO:tasks.rados.rados.0.smithi026.stdout:3237: snap_remove snap 235 2024-09-13T14:30:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:02 smithi026 ceph-mon[97022]: pgmap v641: 121 pgs: 121 active+clean; 623 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.1 MiB/s wr, 12 op/s 2024-09-13T14:30:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:02 smithi026 ceph-mon[97022]: osdmap e691: 8 total, 8 up, 8 in 2024-09-13T14:30:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:02 smithi026 ceph-mon[99461]: pgmap v641: 121 pgs: 121 active+clean; 623 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.1 MiB/s wr, 12 op/s 2024-09-13T14:30:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:02 smithi026 ceph-mon[99461]: osdmap e691: 8 total, 8 up, 8 in 2024-09-13T14:30:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:02 smithi145 ceph-mon[89145]: pgmap v641: 121 pgs: 121 active+clean; 623 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.1 MiB/s wr, 12 op/s 2024-09-13T14:30:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:02 smithi145 ceph-mon[89145]: osdmap e691: 8 total, 8 up, 8 in 2024-09-13T14:30:03.110 INFO:tasks.rados.rados.0.smithi026.stdout:3236: expect (ObjNum 991 snap 309 seq_num 991) 2024-09-13T14:30:03.241 INFO:tasks.rados.rados.0.smithi026.stdout:3236: done (1 left) 2024-09-13T14:30:03.242 INFO:tasks.rados.rados.0.smithi026.stdout:3237: done (0 left) 2024-09-13T14:30:03.242 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 312 2024-09-13T14:30:03.242 INFO:tasks.rados.rados.0.smithi026.stdout:3238: seq_num 1012 ranges {15843328=155648} 2024-09-13T14:30:03.329 INFO:tasks.rados.rados.0.smithi026.stdout:3238: writing smithi02683976-6 from 15843328 to 15998976 tid 1 2024-09-13T14:30:03.329 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 312 2024-09-13T14:30:03.329 INFO:tasks.rados.rados.0.smithi026.stdout:3239: seq_num 1013 ranges {4710400=172032} 2024-09-13T14:30:03.355 INFO:tasks.rados.rados.0.smithi026.stdout:3239: writing smithi02683976-40 from 4710400 to 4882432 tid 1 2024-09-13T14:30:03.355 INFO:tasks.rados.rados.0.smithi026.stdout:3240: snap_remove snap 279 2024-09-13T14:30:03.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:03 smithi026 ceph-mon[97022]: osdmap e692: 8 total, 8 up, 8 in 2024-09-13T14:30:03.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:03 smithi026 ceph-mon[99461]: osdmap e692: 8 total, 8 up, 8 in 2024-09-13T14:30:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:03 smithi145 ceph-mon[89145]: osdmap e692: 8 total, 8 up, 8 in 2024-09-13T14:30:04.007 INFO:tasks.rados.rados.0.smithi026.stdout:3240: done (2 left) 2024-09-13T14:30:04.007 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 312 2024-09-13T14:30:04.007 INFO:tasks.rados.rados.0.smithi026.stdout:3241: seq_num 1014 ranges {3194880=188416} 2024-09-13T14:30:04.029 INFO:tasks.rados.rados.0.smithi026.stdout:3241: writing smithi02683976-17 from 3194880 to 3383296 tid 1 2024-09-13T14:30:04.029 INFO:tasks.rados.rados.0.smithi026.stdout:3242: rollback oid 38 current snap is 312 2024-09-13T14:30:04.029 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 310 2024-09-13T14:30:04.029 INFO:tasks.rados.rados.0.smithi026.stdout:3238: finishing write tid 1 to smithi02683976-6 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3238: finishing write tid 2 to smithi02683976-6 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3238: finishing write tid 3 to smithi02683976-6 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1636 (ObjNum 1012 snap 312 seq_num 1012) dirty exists 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3238: left oid 6 (ObjNum 1012 snap 312 seq_num 1012) 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3239: finishing write tid 1 to smithi02683976-40 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3239: finishing write tid 2 to smithi02683976-40 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3239: finishing write tid 3 to smithi02683976-40 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1622 (ObjNum 1013 snap 312 seq_num 1013) dirty exists 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3239: left oid 40 (ObjNum 1013 snap 312 seq_num 1013) 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3238: done (3 left) 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3239: done (2 left) 2024-09-13T14:30:04.030 INFO:tasks.rados.rados.0.smithi026.stdout:3243: snap_remove snap 311 2024-09-13T14:30:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[97022]: pgmap v644: 121 pgs: 121 active+clean; 628 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-13T14:30:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[97022]: osdmap e693: 8 total, 8 up, 8 in 2024-09-13T14:30:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[97022]: osdmap e694: 8 total, 8 up, 8 in 2024-09-13T14:30:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[99461]: pgmap v644: 121 pgs: 121 active+clean; 628 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-13T14:30:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[99461]: osdmap e693: 8 total, 8 up, 8 in 2024-09-13T14:30:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:04 smithi026 ceph-mon[99461]: osdmap e694: 8 total, 8 up, 8 in 2024-09-13T14:30:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:04 smithi145 ceph-mon[89145]: pgmap v644: 121 pgs: 121 active+clean; 628 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-13T14:30:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:04 smithi145 ceph-mon[89145]: osdmap e693: 8 total, 8 up, 8 in 2024-09-13T14:30:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:04 smithi145 ceph-mon[89145]: osdmap e694: 8 total, 8 up, 8 in 2024-09-13T14:30:05.007 INFO:tasks.rados.rados.0.smithi026.stdout:3241: finishing write tid 1 to smithi02683976-17 2024-09-13T14:30:05.007 INFO:tasks.rados.rados.0.smithi026.stdout:3241: finishing write tid 2 to smithi02683976-17 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3241: finishing write tid 3 to smithi02683976-17 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1223 (ObjNum 1014 snap 312 seq_num 1014) dirty exists 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3241: left oid 17 (ObjNum 1014 snap 312 seq_num 1014) 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3242: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3242: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3242: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1597 (ObjNum 977 snap 306 seq_num 977) dirty exists 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3241: done (2 left) 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3242: done (1 left) 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3243: done (0 left) 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3244: rmattr oid 9 current snap is 312 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3244: done (0 left) 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 312 2024-09-13T14:30:05.008 INFO:tasks.rados.rados.0.smithi026.stdout:3245: seq_num 1015 ranges {4882432=204800} 2024-09-13T14:30:05.035 INFO:tasks.rados.rados.0.smithi026.stdout:3245: writing smithi02683976-40 from 4882432 to 5087232 tid 1 2024-09-13T14:30:05.035 INFO:tasks.rados.rados.0.smithi026.stdout:3246: snap_remove snap 310 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3246: done (1 left) 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3247: copy_from oid 6 from oid 44 current snap is 312 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3245: finishing write tid 1 to smithi02683976-40 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3245: finishing write tid 2 to smithi02683976-40 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3245: finishing write tid 3 to smithi02683976-40 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1624 (ObjNum 1015 snap 312 seq_num 1015) dirty exists 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3245: left oid 40 (ObjNum 1015 snap 312 seq_num 1015) 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3245: done (1 left) 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3248: copy_from oid 33 from oid 49 current snap is 312 2024-09-13T14:30:06.014 INFO:tasks.rados.rados.0.smithi026.stdout:3249: setattr oid 12 current snap is 312 2024-09-13T14:30:06.018 INFO:tasks.rados.rados.0.smithi026.stdout:3250: delete oid 1 current snap is 312 2024-09-13T14:30:06.024 INFO:tasks.rados.rados.0.smithi026.stdout:3247: finishing copy_from to smithi02683976-6 2024-09-13T14:30:06.024 INFO:tasks.rados.rados.0.smithi026.stdout:3247: got expected ENOENT (src dne) 2024-09-13T14:30:06.024 INFO:tasks.rados.rados.0.smithi026.stdout:3247: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:30:06.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1460 (ObjNum 979 snap 307 seq_num 979) dirty exists 2024-09-13T14:30:06.036 INFO:tasks.rados.rados.0.smithi026.stdout:3248: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:30:06.036 INFO:tasks.rados.rados.0.smithi026.stdout:3247: done (3 left) 2024-09-13T14:30:06.036 INFO:tasks.rados.rados.0.smithi026.stdout:3249: done (2 left) 2024-09-13T14:30:06.036 INFO:tasks.rados.rados.0.smithi026.stdout:3250: done (1 left) 2024-09-13T14:30:06.037 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 312 2024-09-13T14:30:06.037 INFO:tasks.rados.rados.0.smithi026.stdout:3251: seq_num 1017 ranges {0=237568} 2024-09-13T14:30:06.038 INFO:tasks.rados.rados.0.smithi026.stdout:3251: writing smithi02683976-44 from 0 to 237568 tid 1 2024-09-13T14:30:06.038 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 312 2024-09-13T14:30:06.038 INFO:tasks.rados.rados.0.smithi026.stdout:3248: finishing copy_from to smithi02683976-33 2024-09-13T14:30:06.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1615 (ObjNum 991 snap 309 seq_num 991) dirty exists 2024-09-13T14:30:06.038 INFO:tasks.rados.rados.0.smithi026.stdout:3252: seq_num 1018 ranges {1286144=253952} 2024-09-13T14:30:06.046 INFO:tasks.rados.rados.0.smithi026.stdout:3252: writing smithi02683976-7 from 1286144 to 1540096 tid 1 2024-09-13T14:30:06.046 INFO:tasks.rados.rados.0.smithi026.stdout:3248: done (2 left) 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:3253: setattr oid 35 current snap is 312 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:3251: finishing write tid 1 to smithi02683976-44 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:3251: finishing write tid 2 to smithi02683976-44 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:3251: finishing write tid 3 to smithi02683976-44 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1435 (ObjNum 1017 snap 312 seq_num 1017) dirty exists 2024-09-13T14:30:06.047 INFO:tasks.rados.rados.0.smithi026.stdout:3251: left oid 44 (ObjNum 1017 snap 312 seq_num 1017) 2024-09-13T14:30:06.048 INFO:tasks.rados.rados.0.smithi026.stdout:3251: done (2 left) 2024-09-13T14:30:06.048 INFO:tasks.rados.rados.0.smithi026.stdout:3254: rollback oid 14 current snap is 312 2024-09-13T14:30:06.048 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 14 to 303 2024-09-13T14:30:06.048 INFO:tasks.rados.rados.0.smithi026.stdout:3255: setattr oid 22 current snap is 312 2024-09-13T14:30:06.049 INFO:tasks.rados.rados.0.smithi026.stdout:3252: finishing write tid 1 to smithi02683976-7 2024-09-13T14:30:06.050 INFO:tasks.rados.rados.0.smithi026.stdout:3252: finishing write tid 2 to smithi02683976-7 2024-09-13T14:30:06.050 INFO:tasks.rados.rados.0.smithi026.stdout:3256: snap_create 2024-09-13T14:30:06.052 INFO:tasks.rados.rados.0.smithi026.stdout:3254: finishing rollback tid 1 to smithi02683976-14 2024-09-13T14:30:06.052 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 1967 (ObjNum 959 snap 300 seq_num 959) dirty exists 2024-09-13T14:30:06.052 INFO:tasks.rados.rados.0.smithi026.stdout:3254: finishing rollback tid 2 to smithi02683976-14 2024-09-13T14:30:06.052 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 1613 (ObjNum 968 snap 302 seq_num 968) dirty exists 2024-09-13T14:30:06.052 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1461 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-13T14:30:06.053 INFO:tasks.rados.rados.0.smithi026.stdout:3252: finishing write tid 3 to smithi02683976-7 2024-09-13T14:30:06.053 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1604 (ObjNum 1018 snap 312 seq_num 1018) dirty exists 2024-09-13T14:30:06.053 INFO:tasks.rados.rados.0.smithi026.stdout:3252: left oid 7 (ObjNum 1018 snap 312 seq_num 1018) 2024-09-13T14:30:06.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:06 smithi145 ceph-mon[89145]: pgmap v647: 121 pgs: 121 active+clean; 624 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-13T14:30:06.257 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:06 smithi145 ceph-mon[89145]: osdmap e695: 8 total, 8 up, 8 in 2024-09-13T14:30:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:06 smithi026 ceph-mon[97022]: pgmap v647: 121 pgs: 121 active+clean; 624 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-13T14:30:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:06 smithi026 ceph-mon[97022]: osdmap e695: 8 total, 8 up, 8 in 2024-09-13T14:30:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:06 smithi026 ceph-mon[99461]: pgmap v647: 121 pgs: 121 active+clean; 624 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-13T14:30:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:06 smithi026 ceph-mon[99461]: osdmap e695: 8 total, 8 up, 8 in 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3252: done (4 left) 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3253: done (3 left) 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3254: done (2 left) 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3255: done (1 left) 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3256: done (0 left) 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 313 2024-09-13T14:30:07.019 INFO:tasks.rados.rados.0.smithi026.stdout:3257: seq_num 1019 ranges {0=270336} 2024-09-13T14:30:07.020 INFO:tasks.rados.rados.0.smithi026.stdout:3257: writing smithi02683976-45 from 0 to 270336 tid 1 2024-09-13T14:30:07.020 INFO:tasks.rados.rados.0.smithi026.stdout:3258: read oid 41 snap 304 2024-09-13T14:30:07.020 INFO:tasks.rados.rados.0.smithi026.stdout:3258: expect (ObjNum 882 snap 275 seq_num 882) 2024-09-13T14:30:07.020 INFO:tasks.rados.rados.0.smithi026.stdout:3259: delete oid 6 current snap is 313 2024-09-13T14:30:07.027 INFO:tasks.rados.rados.0.smithi026.stdout:3259: done (2 left) 2024-09-13T14:30:07.027 INFO:tasks.rados.rados.0.smithi026.stdout:3260: copy_from oid 5 from oid 32 current snap is 313 2024-09-13T14:30:07.027 INFO:tasks.rados.rados.0.smithi026.stdout:3261: setattr oid 24 current snap is 313 2024-09-13T14:30:07.028 INFO:tasks.rados.rados.0.smithi026.stdout:3260: finishing copy_from to smithi02683976-5 2024-09-13T14:30:07.028 INFO:tasks.rados.rados.0.smithi026.stdout:3260: got expected ENOENT (src dne) 2024-09-13T14:30:07.028 INFO:tasks.rados.rados.0.smithi026.stdout:3258: expect (ObjNum 882 snap 275 seq_num 882) 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3257: finishing write tid 1 to smithi02683976-45 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3257: finishing write tid 2 to smithi02683976-45 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3260: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3257: finishing write tid 3 to smithi02683976-45 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1199 (ObjNum 1019 snap 313 seq_num 1019) dirty exists 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3257: left oid 45 (ObjNum 1019 snap 313 seq_num 1019) 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3257: done (3 left) 2024-09-13T14:30:07.039 INFO:tasks.rados.rados.0.smithi026.stdout:3258: done (2 left) 2024-09-13T14:30:07.040 INFO:tasks.rados.rados.0.smithi026.stdout:3260: done (1 left) 2024-09-13T14:30:07.040 INFO:tasks.rados.rados.0.smithi026.stdout:3262: read oid 36 snap 304 2024-09-13T14:30:07.040 INFO:tasks.rados.rados.0.smithi026.stdout:3262: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:07.040 INFO:tasks.rados.rados.0.smithi026.stdout:3263: setattr oid 3 current snap is 313 2024-09-13T14:30:07.041 INFO:tasks.rados.rados.0.smithi026.stdout:3264: read oid 16 snap 312 2024-09-13T14:30:07.041 INFO:tasks.rados.rados.0.smithi026.stdout:3264: expect (ObjNum 987 snap 307 seq_num 987) 2024-09-13T14:30:07.041 INFO:tasks.rados.rados.0.smithi026.stdout:3265: delete oid 20 current snap is 313 2024-09-13T14:30:07.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1305 (ObjNum 904 snap 280 seq_num 904) dirty exists 2024-09-13T14:30:07.049 INFO:tasks.rados.rados.0.smithi026.stdout:3262: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:07.217 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1969 (ObjNum 951 snap 296 seq_num 951) dirty exists 2024-09-13T14:30:07.217 INFO:tasks.rados.rados.0.smithi026.stdout:3264: expect (ObjNum 987 snap 307 seq_num 987) 2024-09-13T14:30:07.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:07 smithi026 ceph-mon[99461]: osdmap e696: 8 total, 8 up, 8 in 2024-09-13T14:30:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:07 smithi026 ceph-mon[97022]: osdmap e696: 8 total, 8 up, 8 in 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3261: done (4 left) 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3262: done (3 left) 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3263: done (2 left) 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3264: done (1 left) 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3265: done (0 left) 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3266: read oid 6 snap -1 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3266: expect deleted 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:3267: rollback oid 47 current snap is 313 2024-09-13T14:30:07.418 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 47 to 303 2024-09-13T14:30:07.419 INFO:tasks.rados.rados.0.smithi026.stdout:3268: snap_create 2024-09-13T14:30:07.419 INFO:tasks.rados.rados.0.smithi026.stdout:3267: finishing rollback tid 1 to smithi02683976-47 2024-09-13T14:30:07.419 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1466 (ObjNum 97 snap 0 seq_num 519668640) dirty dne 2024-09-13T14:30:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:07 smithi145 ceph-mon[89145]: osdmap e696: 8 total, 8 up, 8 in 2024-09-13T14:30:08.021 INFO:tasks.rados.rados.0.smithi026.stdout:3266: done (2 left) 2024-09-13T14:30:08.021 INFO:tasks.rados.rados.0.smithi026.stdout:3267: done (1 left) 2024-09-13T14:30:08.021 INFO:tasks.rados.rados.0.smithi026.stdout:3268: done (0 left) 2024-09-13T14:30:08.022 INFO:tasks.rados.rados.0.smithi026.stdout:3269: rollback oid 24 current snap is 314 2024-09-13T14:30:08.022 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 306 2024-09-13T14:30:08.022 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 314 2024-09-13T14:30:08.022 INFO:tasks.rados.rados.0.smithi026.stdout:3270: seq_num 1022 ranges {0=319488} 2024-09-13T14:30:08.023 INFO:tasks.rados.rados.0.smithi026.stdout:3270: writing smithi02683976-22 from 0 to 319488 tid 1 2024-09-13T14:30:08.023 INFO:tasks.rados.rados.0.smithi026.stdout:3271: delete oid 47 current snap is 314 2024-09-13T14:30:08.028 INFO:tasks.rados.rados.0.smithi026.stdout:3269: finishing rollback tid 0 to smithi02683976-24 2024-09-13T14:30:08.028 INFO:tasks.rados.rados.0.smithi026.stdout:3271: done (2 left) 2024-09-13T14:30:08.028 INFO:tasks.rados.rados.0.smithi026.stdout:3272: copy_from oid 10 from oid 50 current snap is 314 2024-09-13T14:30:08.028 INFO:tasks.rados.rados.0.smithi026.stdout:3273: setattr oid 28 current snap is 314 2024-09-13T14:30:08.028 INFO:tasks.rados.rados.0.smithi026.stdout:3269: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:30:08.029 INFO:tasks.rados.rados.0.smithi026.stdout:3269: finishing rollback tid 2 to smithi02683976-24 2024-09-13T14:30:08.029 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1308 (ObjNum 973 snap 306 seq_num 973) dirty exists 2024-09-13T14:30:08.029 INFO:tasks.rados.rados.0.smithi026.stdout:3272: finishing copy_from to smithi02683976-10 2024-09-13T14:30:08.029 INFO:tasks.rados.rados.0.smithi026.stdout:3272: got expected ENOENT (src dne) 2024-09-13T14:30:08.029 INFO:tasks.rados.rados.0.smithi026.stdout:3272: finishing copy_from racing read to smithi02683976-10 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:3269: done (3 left) 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:3272: done (2 left) 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:3274: rollback oid 29 current snap is 314 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 306 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:3275: rmattr oid 12 current snap is 314 2024-09-13T14:30:08.031 INFO:tasks.rados.rados.0.smithi026.stdout:3276: rollback oid 48 current snap is 314 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 298 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:3277: copy_from oid 36 from oid 1 current snap is 314 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:3278: rollback oid 42 current snap is 314 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 42 to 301 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 314 2024-09-13T14:30:08.032 INFO:tasks.rados.rados.0.smithi026.stdout:3279: seq_num 1024 ranges {0=352256} 2024-09-13T14:30:08.034 INFO:tasks.rados.rados.0.smithi026.stdout:3279: writing smithi02683976-25 from 0 to 352256 tid 1 2024-09-13T14:30:08.034 INFO:tasks.rados.rados.0.smithi026.stdout:3270: finishing write tid 1 to smithi02683976-22 2024-09-13T14:30:08.034 INFO:tasks.rados.rados.0.smithi026.stdout:3280: read oid 30 snap 291 2024-09-13T14:30:08.034 INFO:tasks.rados.rados.0.smithi026.stdout:3280: expect (ObjNum 855 snap 263 seq_num 855) 2024-09-13T14:30:08.034 INFO:tasks.rados.rados.0.smithi026.stdout:3281: read oid 7 snap -1 2024-09-13T14:30:08.035 INFO:tasks.rados.rados.0.smithi026.stdout:3281: expect (ObjNum 1018 snap 312 seq_num 1018) 2024-09-13T14:30:08.035 INFO:tasks.rados.rados.0.smithi026.stdout:3282: delete oid 13 current snap is 314 2024-09-13T14:30:08.035 INFO:tasks.rados.rados.0.smithi026.stdout:3270: finishing write tid 2 to smithi02683976-22 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3277: finishing copy_from to smithi02683976-36 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3277: got expected ENOENT (src dne) 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3277: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3270: finishing write tid 3 to smithi02683976-22 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1463 (ObjNum 1022 snap 314 seq_num 1022) dirty exists 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3270: left oid 22 (ObjNum 1022 snap 314 seq_num 1022) 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1464 (ObjNum 979 snap 307 seq_num 979) dirty exists 2024-09-13T14:30:08.036 INFO:tasks.rados.rados.0.smithi026.stdout:3276: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:30:08.038 INFO:tasks.rados.rados.0.smithi026.stdout:3274: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:30:08.038 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 1595 (ObjNum 33 snap 0 seq_num 520048096) dirty dne 2024-09-13T14:30:08.042 INFO:tasks.rados.rados.0.smithi026.stdout:3276: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:30:08.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1470 (ObjNum 49 snap 0 seq_num 519927008) dirty dne 2024-09-13T14:30:08.042 INFO:tasks.rados.rados.0.smithi026.stdout:3278: finishing rollback tid 0 to smithi02683976-42 2024-09-13T14:30:08.042 INFO:tasks.rados.rados.0.smithi026.stdout:3278: finishing rollback tid 1 to smithi02683976-42 2024-09-13T14:30:08.043 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1625 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:08.043 INFO:tasks.rados.rados.0.smithi026.stdout:3279: finishing write tid 1 to smithi02683976-25 2024-09-13T14:30:08.043 INFO:tasks.rados.rados.0.smithi026.stdout:3279: finishing write tid 2 to smithi02683976-25 2024-09-13T14:30:08.044 INFO:tasks.rados.rados.0.smithi026.stdout:3278: finishing rollback tid 2 to smithi02683976-42 2024-09-13T14:30:08.044 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1468 (ObjNum 956 snap 296 seq_num 956) dirty exists 2024-09-13T14:30:08.044 INFO:tasks.rados.rados.0.smithi026.stdout:3281: expect (ObjNum 1018 snap 312 seq_num 1018) 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:3279: finishing write tid 3 to smithi02683976-25 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1310 (ObjNum 1024 snap 314 seq_num 1024) dirty exists 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:3279: left oid 25 (ObjNum 1024 snap 314 seq_num 1024) 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:3270: done (10 left) 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:3273: done (9 left) 2024-09-13T14:30:08.107 INFO:tasks.rados.rados.0.smithi026.stdout:3274: done (8 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3275: done (7 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3276: done (6 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3277: done (5 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3278: done (4 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3279: done (3 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3281: done (2 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3282: done (1 left) 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3283: delete oid 33 current snap is 314 2024-09-13T14:30:08.108 INFO:tasks.rados.rados.0.smithi026.stdout:3280: expect (ObjNum 855 snap 263 seq_num 855) 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3280: done (1 left) 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3283: done (0 left) 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3284: read oid 45 snap -1 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3284: expect (ObjNum 1019 snap 313 seq_num 1019) 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3285: copy_from oid 44 from oid 31 current snap is 314 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:3286: rollback oid 4 current snap is 314 2024-09-13T14:30:08.170 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 306 2024-09-13T14:30:08.171 INFO:tasks.rados.rados.0.smithi026.stdout:3287: snap_create 2024-09-13T14:30:08.173 INFO:tasks.rados.rados.0.smithi026.stdout:3286: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:30:08.173 INFO:tasks.rados.rados.0.smithi026.stdout:3286: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:30:08.173 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1819 (ObjNum 953 snap 296 seq_num 953) dirty exists 2024-09-13T14:30:08.174 INFO:tasks.rados.rados.0.smithi026.stdout:3284: expect (ObjNum 1019 snap 313 seq_num 1019) 2024-09-13T14:30:08.182 INFO:tasks.rados.rados.0.smithi026.stdout:3285: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:30:08.182 INFO:tasks.rados.rados.0.smithi026.stdout:3285: finishing copy_from to smithi02683976-44 2024-09-13T14:30:08.182 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1633 (ObjNum 963 snap 302 seq_num 963) dirty exists 2024-09-13T14:30:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:08 smithi026 ceph-mon[97022]: pgmap v650: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 734 KiB/s wr, 15 op/s 2024-09-13T14:30:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:08 smithi026 ceph-mon[97022]: osdmap e697: 8 total, 8 up, 8 in 2024-09-13T14:30:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:08 smithi026 ceph-mon[99461]: pgmap v650: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 734 KiB/s wr, 15 op/s 2024-09-13T14:30:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:08 smithi026 ceph-mon[99461]: osdmap e697: 8 total, 8 up, 8 in 2024-09-13T14:30:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:08 smithi145 ceph-mon[89145]: pgmap v650: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 734 KiB/s wr, 15 op/s 2024-09-13T14:30:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:08 smithi145 ceph-mon[89145]: osdmap e697: 8 total, 8 up, 8 in 2024-09-13T14:30:09.007 INFO:tasks.rados.rados.0.smithi026.stdout:3284: done (3 left) 2024-09-13T14:30:09.007 INFO:tasks.rados.rados.0.smithi026.stdout:3285: done (2 left) 2024-09-13T14:30:09.008 INFO:tasks.rados.rados.0.smithi026.stdout:3286: done (1 left) 2024-09-13T14:30:09.008 INFO:tasks.rados.rados.0.smithi026.stdout:3287: done (0 left) 2024-09-13T14:30:09.008 INFO:tasks.rados.rados.0.smithi026.stdout:3288: snap_create 2024-09-13T14:30:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:09 smithi026 ceph-mon[97022]: osdmap e698: 8 total, 8 up, 8 in 2024-09-13T14:30:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:09 smithi026 ceph-mon[97022]: osdmap e699: 8 total, 8 up, 8 in 2024-09-13T14:30:09.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:09 smithi026 ceph-mon[99461]: osdmap e698: 8 total, 8 up, 8 in 2024-09-13T14:30:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:09 smithi026 ceph-mon[99461]: osdmap e699: 8 total, 8 up, 8 in 2024-09-13T14:30:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:09 smithi145 ceph-mon[89145]: osdmap e698: 8 total, 8 up, 8 in 2024-09-13T14:30:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:09 smithi145 ceph-mon[89145]: osdmap e699: 8 total, 8 up, 8 in 2024-09-13T14:30:10.008 INFO:tasks.rados.rados.0.smithi026.stdout:3288: done (0 left) 2024-09-13T14:30:10.008 INFO:tasks.rados.rados.0.smithi026.stdout:3289: delete oid 22 current snap is 316 2024-09-13T14:30:10.014 INFO:tasks.rados.rados.0.smithi026.stdout:3289: done (0 left) 2024-09-13T14:30:10.014 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 316 2024-09-13T14:30:10.014 INFO:tasks.rados.rados.0.smithi026.stdout:3290: seq_num 1028 ranges {0=425984} 2024-09-13T14:30:10.016 INFO:tasks.rados.rados.0.smithi026.stdout:3290: writing smithi02683976-47 from 0 to 425984 tid 1 2024-09-13T14:30:10.016 INFO:tasks.rados.rados.0.smithi026.stdout:3291: setattr oid 7 current snap is 316 2024-09-13T14:30:10.018 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 2 current snap is 316 2024-09-13T14:30:10.018 INFO:tasks.rados.rados.0.smithi026.stdout:3292: seq_num 1029 ranges {0=442368} 2024-09-13T14:30:10.021 INFO:tasks.rados.rados.0.smithi026.stdout:3292: writing smithi02683976-2 from 0 to 442368 tid 1 2024-09-13T14:30:10.021 INFO:tasks.rados.rados.0.smithi026.stdout:3293: rmattr oid 28 current snap is 316 2024-09-13T14:30:10.021 INFO:tasks.rados.rados.0.smithi026.stdout:3290: finishing write tid 1 to smithi02683976-47 2024-09-13T14:30:10.021 INFO:tasks.rados.rados.0.smithi026.stdout:3294: snap_create 2024-09-13T14:30:10.021 INFO:tasks.rados.rados.0.smithi026.stdout:3290: finishing write tid 2 to smithi02683976-47 2024-09-13T14:30:10.023 INFO:tasks.rados.rados.0.smithi026.stdout:3290: finishing write tid 3 to smithi02683976-47 2024-09-13T14:30:10.023 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1472 (ObjNum 1028 snap 316 seq_num 1028) dirty exists 2024-09-13T14:30:10.023 INFO:tasks.rados.rados.0.smithi026.stdout:3290: left oid 47 (ObjNum 1028 snap 316 seq_num 1028) 2024-09-13T14:30:10.025 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1605 (ObjNum 1018 snap 312 seq_num 1018) dirty exists 2024-09-13T14:30:10.031 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1626 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:10.031 INFO:tasks.rados.rados.0.smithi026.stdout:3292: finishing write tid 1 to smithi02683976-2 2024-09-13T14:30:10.032 INFO:tasks.rados.rados.0.smithi026.stdout:3292: finishing write tid 2 to smithi02683976-2 2024-09-13T14:30:10.033 INFO:tasks.rados.rados.0.smithi026.stdout:3292: finishing write tid 3 to smithi02683976-2 2024-09-13T14:30:10.033 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 1312 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:30:10.033 INFO:tasks.rados.rados.0.smithi026.stdout:3292: left oid 2 (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:30:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:10 smithi026 ceph-mon[97022]: pgmap v653: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 758 KiB/s wr, 20 op/s 2024-09-13T14:30:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:10 smithi026 ceph-mon[97022]: osdmap e700: 8 total, 8 up, 8 in 2024-09-13T14:30:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:10 smithi026 ceph-mon[99461]: pgmap v653: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 758 KiB/s wr, 20 op/s 2024-09-13T14:30:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:10 smithi026 ceph-mon[99461]: osdmap e700: 8 total, 8 up, 8 in 2024-09-13T14:30:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:10 smithi145 ceph-mon[89145]: pgmap v653: 121 pgs: 121 active+clean; 567 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 758 KiB/s wr, 20 op/s 2024-09-13T14:30:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:10 smithi145 ceph-mon[89145]: osdmap e700: 8 total, 8 up, 8 in 2024-09-13T14:30:10.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3290: done (4 left) 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3291: done (3 left) 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3292: done (2 left) 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3293: done (1 left) 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3294: done (0 left) 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3295: copy_from oid 26 from oid 49 current snap is 317 2024-09-13T14:30:11.010 INFO:tasks.rados.rados.0.smithi026.stdout:3296: setattr oid 24 current snap is 317 2024-09-13T14:30:11.012 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 317 2024-09-13T14:30:11.012 INFO:tasks.rados.rados.0.smithi026.stdout:3297: seq_num 1030 ranges {4808704=458752} 2024-09-13T14:30:11.041 INFO:tasks.rados.rados.0.smithi026.stdout:3297: writing smithi02683976-16 from 4808704 to 5267456 tid 1 2024-09-13T14:30:11.041 INFO:tasks.rados.rados.0.smithi026.stdout:3298: delete oid 41 current snap is 317 2024-09-13T14:30:11.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1313 (ObjNum 973 snap 306 seq_num 973) dirty exists 2024-09-13T14:30:11.042 INFO:tasks.rados.rados.0.smithi026.stdout:3295: finishing copy_from racing read to smithi02683976-26 2024-09-13T14:30:11.042 INFO:tasks.rados.rados.0.smithi026.stdout:3295: finishing copy_from to smithi02683976-26 2024-09-13T14:30:11.042 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 1615 (ObjNum 991 snap 309 seq_num 991) dirty exists 2024-09-13T14:30:11.044 INFO:tasks.rados.rados.0.smithi026.stdout:3295: done (3 left) 2024-09-13T14:30:11.044 INFO:tasks.rados.rados.0.smithi026.stdout:3296: done (2 left) 2024-09-13T14:30:11.044 INFO:tasks.rados.rados.0.smithi026.stdout:3298: done (1 left) 2024-09-13T14:30:11.044 INFO:tasks.rados.rados.0.smithi026.stdout:3299: rmattr oid 10 current snap is 317 2024-09-13T14:30:11.044 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 317 2024-09-13T14:30:11.045 INFO:tasks.rados.rados.0.smithi026.stdout:3300: seq_num 1032 ranges {12763136=491520} 2024-09-13T14:30:11.117 INFO:tasks.rados.rados.0.smithi026.stdout:3300: writing smithi02683976-3 from 12763136 to 13254656 tid 1 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:3301: snap_create 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:3297: finishing write tid 1 to smithi02683976-16 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:3297: finishing write tid 2 to smithi02683976-16 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1627 (ObjNum 835 snap 260 seq_num 835) dirty exists 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:3297: finishing write tid 3 to smithi02683976-16 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1647 (ObjNum 1030 snap 317 seq_num 1030) dirty exists 2024-09-13T14:30:11.118 INFO:tasks.rados.rados.0.smithi026.stdout:3297: left oid 16 (ObjNum 1030 snap 317 seq_num 1030) 2024-09-13T14:30:11.121 INFO:tasks.rados.rados.0.smithi026.stdout:3300: finishing write tid 1 to smithi02683976-3 2024-09-13T14:30:11.122 INFO:tasks.rados.rados.0.smithi026.stdout:3300: finishing write tid 2 to smithi02683976-3 2024-09-13T14:30:11.123 INFO:tasks.rados.rados.0.smithi026.stdout:3300: finishing write tid 3 to smithi02683976-3 2024-09-13T14:30:11.123 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1974 (ObjNum 1032 snap 317 seq_num 1032) dirty exists 2024-09-13T14:30:11.123 INFO:tasks.rados.rados.0.smithi026.stdout:3300: left oid 3 (ObjNum 1032 snap 317 seq_num 1032) 2024-09-13T14:30:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:11] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:30:12.035 INFO:tasks.rados.rados.0.smithi026.stdout:3297: done (3 left) 2024-09-13T14:30:12.035 INFO:tasks.rados.rados.0.smithi026.stdout:3299: done (2 left) 2024-09-13T14:30:12.035 INFO:tasks.rados.rados.0.smithi026.stdout:3300: done (1 left) 2024-09-13T14:30:12.035 INFO:tasks.rados.rados.0.smithi026.stdout:3301: done (0 left) 2024-09-13T14:30:12.036 INFO:tasks.rados.rados.0.smithi026.stdout:3302: delete oid 25 current snap is 318 2024-09-13T14:30:12.038 INFO:tasks.rados.rados.0.smithi026.stdout:3302: done (0 left) 2024-09-13T14:30:12.038 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 318 2024-09-13T14:30:12.038 INFO:tasks.rados.rados.0.smithi026.stdout:3303: seq_num 1034 ranges {270336=524288} 2024-09-13T14:30:12.042 INFO:tasks.rados.rados.0.smithi026.stdout:3303: writing smithi02683976-45 from 270336 to 794624 tid 1 2024-09-13T14:30:12.043 INFO:tasks.rados.rados.0.smithi026.stdout:3304: read oid 17 snap -1 2024-09-13T14:30:12.043 INFO:tasks.rados.rados.0.smithi026.stdout:3304: expect (ObjNum 1014 snap 312 seq_num 1014) 2024-09-13T14:30:12.043 INFO:tasks.rados.rados.0.smithi026.stdout:3305: delete oid 32 current snap is 318 2024-09-13T14:30:12.047 INFO:tasks.rados.rados.0.smithi026.stdout:3305: done (2 left) 2024-09-13T14:30:12.047 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 318 2024-09-13T14:30:12.047 INFO:tasks.rados.rados.0.smithi026.stdout:3306: seq_num 1036 ranges {0=557056} 2024-09-13T14:30:12.050 INFO:tasks.rados.rados.0.smithi026.stdout:3306: writing smithi02683976-1 from 0 to 557056 tid 1 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:3307: snap_create 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:3303: finishing write tid 1 to smithi02683976-45 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:3303: finishing write tid 2 to smithi02683976-45 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:3303: finishing write tid 3 to smithi02683976-45 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1201 (ObjNum 1034 snap 318 seq_num 1034) dirty exists 2024-09-13T14:30:12.051 INFO:tasks.rados.rados.0.smithi026.stdout:3303: left oid 45 (ObjNum 1034 snap 318 seq_num 1034) 2024-09-13T14:30:12.055 INFO:tasks.rados.rados.0.smithi026.stdout:3306: finishing write tid 1 to smithi02683976-1 2024-09-13T14:30:12.055 INFO:tasks.rados.rados.0.smithi026.stdout:3306: finishing write tid 2 to smithi02683976-1 2024-09-13T14:30:12.057 INFO:tasks.rados.rados.0.smithi026.stdout:3306: finishing write tid 3 to smithi02683976-1 2024-09-13T14:30:12.057 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1976 (ObjNum 1036 snap 318 seq_num 1036) dirty exists 2024-09-13T14:30:12.058 INFO:tasks.rados.rados.0.smithi026.stdout:3306: left oid 1 (ObjNum 1036 snap 318 seq_num 1036) 2024-09-13T14:30:12.060 INFO:tasks.rados.rados.0.smithi026.stdout:3304: expect (ObjNum 1014 snap 312 seq_num 1014) 2024-09-13T14:30:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[97022]: pgmap v656: 121 pgs: 121 active+clean; 590 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.7 MiB/s wr, 47 op/s 2024-09-13T14:30:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[97022]: osdmap e701: 8 total, 8 up, 8 in 2024-09-13T14:30:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[99461]: pgmap v656: 121 pgs: 121 active+clean; 590 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.7 MiB/s wr, 47 op/s 2024-09-13T14:30:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[99461]: osdmap e701: 8 total, 8 up, 8 in 2024-09-13T14:30:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:12 smithi145 ceph-mon[89145]: pgmap v656: 121 pgs: 121 active+clean; 590 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.7 MiB/s wr, 47 op/s 2024-09-13T14:30:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:12 smithi145 ceph-mon[89145]: osdmap e701: 8 total, 8 up, 8 in 2024-09-13T14:30:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:13.041 INFO:tasks.rados.rados.0.smithi026.stdout:3303: done (3 left) 2024-09-13T14:30:13.041 INFO:tasks.rados.rados.0.smithi026.stdout:3304: done (2 left) 2024-09-13T14:30:13.042 INFO:tasks.rados.rados.0.smithi026.stdout:3306: done (1 left) 2024-09-13T14:30:13.042 INFO:tasks.rados.rados.0.smithi026.stdout:3307: done (0 left) 2024-09-13T14:30:13.042 INFO:tasks.rados.rados.0.smithi026.stdout:3308: delete oid 26 current snap is 319 2024-09-13T14:30:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:3308: done (0 left) 2024-09-13T14:30:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:3309: copy_from oid 40 from oid 42 current snap is 319 2024-09-13T14:30:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:3310: rmattr oid 28 current snap is 319 2024-09-13T14:30:13.046 INFO:tasks.rados.rados.0.smithi026.stdout:3311: delete oid 50 current snap is 319 2024-09-13T14:30:13.051 INFO:tasks.rados.rados.0.smithi026.stdout:3311: done (2 left) 2024-09-13T14:30:13.051 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 319 2024-09-13T14:30:13.051 INFO:tasks.rados.rados.0.smithi026.stdout:3312: seq_num 1039 ranges {0=606208} 2024-09-13T14:30:13.055 INFO:tasks.rados.rados.0.smithi026.stdout:3312: writing smithi02683976-6 from 0 to 606208 tid 1 2024-09-13T14:30:13.055 INFO:tasks.rados.rados.0.smithi026.stdout:3313: read oid 44 snap -1 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:3313: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:3314: read oid 30 snap -1 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:3314: expect deleted 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 319 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1628 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:13.056 INFO:tasks.rados.rados.0.smithi026.stdout:3315: seq_num 1040 ranges {3194880=622592} 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3315: writing smithi02683976-46 from 3194880 to 3817472 tid 1 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3310: done (5 left) 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3316: rollback oid 8 current snap is 319 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 309 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3312: finishing write tid 1 to smithi02683976-6 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3309: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3312: finishing write tid 2 to smithi02683976-6 2024-09-13T14:30:13.078 INFO:tasks.rados.rados.0.smithi026.stdout:3312: finishing write tid 3 to smithi02683976-6 2024-09-13T14:30:13.079 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1648 (ObjNum 1039 snap 319 seq_num 1039) dirty exists 2024-09-13T14:30:13.079 INFO:tasks.rados.rados.0.smithi026.stdout:3312: left oid 6 (ObjNum 1039 snap 319 seq_num 1039) 2024-09-13T14:30:13.079 INFO:tasks.rados.rados.0.smithi026.stdout:3309: finishing copy_from to smithi02683976-40 2024-09-13T14:30:13.079 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1468 (ObjNum 956 snap 296 seq_num 956) dirty exists 2024-09-13T14:30:13.079 INFO:tasks.rados.rados.0.smithi026.stdout:3313: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:30:13.125 INFO:tasks.rados.rados.0.smithi026.stdout:3316: finishing rollback tid 0 to smithi02683976-8 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3316: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3315: finishing write tid 1 to smithi02683976-46 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3316: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1519 (ObjNum 943 snap 294 seq_num 943) dirty exists 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3315: finishing write tid 2 to smithi02683976-46 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3309: done (5 left) 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3312: done (4 left) 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3313: done (3 left) 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3314: done (2 left) 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3316: done (1 left) 2024-09-13T14:30:13.126 INFO:tasks.rados.rados.0.smithi026.stdout:3317: rollback oid 18 current snap is 319 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 314 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:3315: finishing write tid 3 to smithi02683976-46 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1635 (ObjNum 1040 snap 319 seq_num 1040) dirty exists 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:3315: left oid 46 (ObjNum 1040 snap 319 seq_num 1040) 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:3315: done (1 left) 2024-09-13T14:30:13.127 INFO:tasks.rados.rados.0.smithi026.stdout:3318: snap_create 2024-09-13T14:30:13.128 INFO:tasks.rados.rados.0.smithi026.stdout:3317: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:30:13.129 INFO:tasks.rados.rados.0.smithi026.stdout:3317: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:30:13.129 INFO:tasks.rados.rados.0.smithi026.stdout:3317: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:30:13.129 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1618 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-13T14:30:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:13 smithi026 ceph-mon[97022]: osdmap e702: 8 total, 8 up, 8 in 2024-09-13T14:30:13.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:13 smithi026 ceph-mon[99461]: osdmap e702: 8 total, 8 up, 8 in 2024-09-13T14:30:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:13 smithi145 ceph-mon[89145]: osdmap e702: 8 total, 8 up, 8 in 2024-09-13T14:30:14.009 INFO:tasks.rados.rados.0.smithi026.stdout:3317: done (1 left) 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3318: done (0 left) 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3319: read oid 15 snap -1 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3319: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3320: read oid 45 snap -1 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3320: expect (ObjNum 1034 snap 318 seq_num 1034) 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3321: rollback oid 44 current snap is 320 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 306 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 320 2024-09-13T14:30:14.010 INFO:tasks.rados.rados.0.smithi026.stdout:3322: seq_num 1041 ranges {557056=638976} 2024-09-13T14:30:14.017 INFO:tasks.rados.rados.0.smithi026.stdout:3322: writing smithi02683976-1 from 557056 to 1196032 tid 1 2024-09-13T14:30:14.018 INFO:tasks.rados.rados.0.smithi026.stdout:3323: read oid 34 snap -1 2024-09-13T14:30:14.018 INFO:tasks.rados.rados.0.smithi026.stdout:3323: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:30:14.019 INFO:tasks.rados.rados.0.smithi026.stdout:3320: expect (ObjNum 1034 snap 318 seq_num 1034) 2024-09-13T14:30:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3321: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3321: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3321: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1638 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3322: finishing write tid 1 to smithi02683976-1 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3322: finishing write tid 2 to smithi02683976-1 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3322: finishing write tid 3 to smithi02683976-1 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 1979 (ObjNum 1041 snap 320 seq_num 1041) dirty exists 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3322: left oid 1 (ObjNum 1041 snap 320 seq_num 1041) 2024-09-13T14:30:14.051 INFO:tasks.rados.rados.0.smithi026.stdout:3319: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:14.216 INFO:tasks.rados.rados.0.smithi026.stdout:3323: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:30:14.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[99461]: pgmap v659: 121 pgs: 121 active+clean; 593 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.9 MiB/s wr, 51 op/s 2024-09-13T14:30:14.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[99461]: osdmap e703: 8 total, 8 up, 8 in 2024-09-13T14:30:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[99461]: osdmap e704: 8 total, 8 up, 8 in 2024-09-13T14:30:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[97022]: pgmap v659: 121 pgs: 121 active+clean; 593 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.9 MiB/s wr, 51 op/s 2024-09-13T14:30:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[97022]: osdmap e703: 8 total, 8 up, 8 in 2024-09-13T14:30:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:14 smithi026 ceph-mon[97022]: osdmap e704: 8 total, 8 up, 8 in 2024-09-13T14:30:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:14 smithi145 ceph-mon[89145]: pgmap v659: 121 pgs: 121 active+clean; 593 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.9 MiB/s wr, 51 op/s 2024-09-13T14:30:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:14 smithi145 ceph-mon[89145]: osdmap e703: 8 total, 8 up, 8 in 2024-09-13T14:30:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:14 smithi145 ceph-mon[89145]: osdmap e704: 8 total, 8 up, 8 in 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3319: done (4 left) 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3320: done (3 left) 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3321: done (2 left) 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3322: done (1 left) 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3323: done (0 left) 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:3324: rollback oid 18 current snap is 320 2024-09-13T14:30:15.889 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 309 2024-09-13T14:30:15.890 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 320 2024-09-13T14:30:15.890 INFO:tasks.rados.rados.0.smithi026.stdout:3325: seq_num 1042 ranges {3121152=655360} 2024-09-13T14:30:15.910 INFO:tasks.rados.rados.0.smithi026.stdout:3325: writing smithi02683976-27 from 3121152 to 3776512 tid 1 2024-09-13T14:30:15.910 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 320 2024-09-13T14:30:15.910 INFO:tasks.rados.rados.0.smithi026.stdout:3326: seq_num 1043 ranges {0=671744} 2024-09-13T14:30:15.914 INFO:tasks.rados.rados.0.smithi026.stdout:3326: writing smithi02683976-41 from 0 to 671744 tid 1 2024-09-13T14:30:15.914 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 320 2024-09-13T14:30:15.914 INFO:tasks.rados.rados.0.smithi026.stdout:3327: seq_num 1044 ranges {7233536=688128} 2024-09-13T14:30:15.957 INFO:tasks.rados.rados.0.smithi026.stdout:3327: writing smithi02683976-38 from 7233536 to 7921664 tid 1 2024-09-13T14:30:15.957 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 320 2024-09-13T14:30:15.957 INFO:tasks.rados.rados.0.smithi026.stdout:3324: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:30:15.957 INFO:tasks.rados.rados.0.smithi026.stdout:3324: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:30:15.958 INFO:tasks.rados.rados.0.smithi026.stdout:3324: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:30:15.958 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1622 (ObjNum 129 snap 0 seq_num 519867536) dirty exists 2024-09-13T14:30:15.958 INFO:tasks.rados.rados.0.smithi026.stdout:3325: finishing write tid 1 to smithi02683976-27 2024-09-13T14:30:15.958 INFO:tasks.rados.rados.0.smithi026.stdout:3328: seq_num 1045 ranges {0=704512} 2024-09-13T14:30:15.961 INFO:tasks.rados.rados.0.smithi026.stdout:3328: writing smithi02683976-33 from 0 to 704512 tid 1 2024-09-13T14:30:15.961 INFO:tasks.rados.rados.0.smithi026.stdout:3324: done (4 left) 2024-09-13T14:30:15.961 INFO:tasks.rados.rados.0.smithi026.stdout:3329: snap_remove snap 312 2024-09-13T14:30:16.050 INFO:tasks.rados.rados.0.smithi026.stdout:3329: done (4 left) 2024-09-13T14:30:16.050 INFO:tasks.rados.rados.0.smithi026.stdout:3330: read oid 24 snap -1 2024-09-13T14:30:16.050 INFO:tasks.rados.rados.0.smithi026.stdout:3330: expect (ObjNum 973 snap 306 seq_num 973) 2024-09-13T14:30:16.050 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 320 2024-09-13T14:30:16.051 INFO:tasks.rados.rados.0.smithi026.stdout:3331: seq_num 1046 ranges {2859008=729088} 2024-09-13T14:30:16.072 INFO:tasks.rados.rados.0.smithi026.stdout:3331: writing smithi02683976-42 from 2859008 to 3588096 tid 1 2024-09-13T14:30:16.072 INFO:tasks.rados.rados.0.smithi026.stdout:3332: rmattr oid 36 current snap is 320 2024-09-13T14:30:16.072 INFO:tasks.rados.rados.0.smithi026.stdout:3332: done (6 left) 2024-09-13T14:30:16.072 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 320 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3325: finishing write tid 2 to smithi02683976-27 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3325: finishing write tid 3 to smithi02683976-27 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1982 (ObjNum 1042 snap 320 seq_num 1042) dirty exists 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3325: left oid 27 (ObjNum 1042 snap 320 seq_num 1042) 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3326: finishing write tid 1 to smithi02683976-41 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3326: finishing write tid 2 to smithi02683976-41 2024-09-13T14:30:16.073 INFO:tasks.rados.rados.0.smithi026.stdout:3333: seq_num 1047 ranges {425984=745472} 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3333: writing smithi02683976-47 from 425984 to 1171456 tid 1 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3325: done (6 left) 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 320 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3326: finishing write tid 3 to smithi02683976-41 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1226 (ObjNum 1043 snap 320 seq_num 1043) dirty exists 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3326: left oid 41 (ObjNum 1043 snap 320 seq_num 1043) 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3327: finishing write tid 1 to smithi02683976-38 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3327: finishing write tid 2 to smithi02683976-38 2024-09-13T14:30:16.080 INFO:tasks.rados.rados.0.smithi026.stdout:3334: seq_num 1048 ranges {0=761856} 2024-09-13T14:30:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3334: writing smithi02683976-11 from 0 to 761856 tid 1 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3326: done (6 left) 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3335: delete oid 35 current snap is 320 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3327: finishing write tid 3 to smithi02683976-38 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1625 (ObjNum 1044 snap 320 seq_num 1044) dirty exists 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3327: left oid 38 (ObjNum 1044 snap 320 seq_num 1044) 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3328: finishing write tid 1 to smithi02683976-33 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3328: finishing write tid 2 to smithi02683976-33 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3328: finishing write tid 3 to smithi02683976-33 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1984 (ObjNum 1045 snap 320 seq_num 1045) dirty exists 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3328: left oid 33 (ObjNum 1045 snap 320 seq_num 1045) 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3331: finishing write tid 1 to smithi02683976-42 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3331: finishing write tid 2 to smithi02683976-42 2024-09-13T14:30:16.085 INFO:tasks.rados.rados.0.smithi026.stdout:3330: expect (ObjNum 973 snap 306 seq_num 973) 2024-09-13T14:30:16.266 INFO:tasks.rados.rados.0.smithi026.stdout:3331: finishing write tid 3 to smithi02683976-42 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1470 (ObjNum 1046 snap 320 seq_num 1046) dirty exists 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3331: left oid 42 (ObjNum 1046 snap 320 seq_num 1046) 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3333: finishing write tid 1 to smithi02683976-47 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3333: finishing write tid 2 to smithi02683976-47 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3333: finishing write tid 3 to smithi02683976-47 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1474 (ObjNum 1047 snap 320 seq_num 1047) dirty exists 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3333: left oid 47 (ObjNum 1047 snap 320 seq_num 1047) 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3334: finishing write tid 1 to smithi02683976-11 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3334: finishing write tid 2 to smithi02683976-11 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3334: finishing write tid 3 to smithi02683976-11 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1652 (ObjNum 1048 snap 320 seq_num 1048) dirty exists 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3334: left oid 11 (ObjNum 1048 snap 320 seq_num 1048) 2024-09-13T14:30:16.267 INFO:tasks.rados.rados.0.smithi026.stdout:3327: done (6 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3328: done (5 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3330: done (4 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3331: done (3 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3333: done (2 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3334: done (1 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3335: done (0 left) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3336: read oid 15 snap 306 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3336: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3337: copy_from oid 38 from oid 18 current snap is 320 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 320 2024-09-13T14:30:16.268 INFO:tasks.rados.rados.0.smithi026.stdout:3338: seq_num 1050 ranges {0=794624} 2024-09-13T14:30:16.271 INFO:tasks.rados.rados.0.smithi026.stdout:3338: writing smithi02683976-48 from 0 to 794624 tid 1 2024-09-13T14:30:16.271 INFO:tasks.rados.rados.0.smithi026.stdout:3339: read oid 42 snap 309 2024-09-13T14:30:16.271 INFO:tasks.rados.rados.0.smithi026.stdout:3339: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-13T14:30:16.271 INFO:tasks.rados.rados.0.smithi026.stdout:3340: snap_remove snap 315 2024-09-13T14:30:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:16 smithi026 ceph-mon[97022]: pgmap v662: 121 pgs: 121 active+clean; 594 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 454 KiB/s wr, 16 op/s 2024-09-13T14:30:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:16 smithi026 ceph-mon[99461]: pgmap v662: 121 pgs: 121 active+clean; 594 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 454 KiB/s wr, 16 op/s 2024-09-13T14:30:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:16 smithi145 ceph-mon[89145]: pgmap v662: 121 pgs: 121 active+clean; 594 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 454 KiB/s wr, 16 op/s 2024-09-13T14:30:17.057 INFO:tasks.rados.rados.0.smithi026.stdout:3340: done (4 left) 2024-09-13T14:30:17.057 INFO:tasks.rados.rados.0.smithi026.stdout:3341: snap_create 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3337: finishing copy_from racing read to smithi02683976-38 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3337: finishing copy_from to smithi02683976-38 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1622 (ObjNum 113 snap 0 seq_num 519966288) dirty exists 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3338: finishing write tid 1 to smithi02683976-48 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3338: finishing write tid 2 to smithi02683976-48 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3338: finishing write tid 3 to smithi02683976-48 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1476 (ObjNum 1050 snap 320 seq_num 1050) dirty exists 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3338: left oid 48 (ObjNum 1050 snap 320 seq_num 1050) 2024-09-13T14:30:17.058 INFO:tasks.rados.rados.0.smithi026.stdout:3339: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-13T14:30:17.159 INFO:tasks.rados.rados.0.smithi026.stdout:3336: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:17 smithi026 ceph-mon[97022]: osdmap e705: 8 total, 8 up, 8 in 2024-09-13T14:30:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:17 smithi026 ceph-mon[99461]: osdmap e705: 8 total, 8 up, 8 in 2024-09-13T14:30:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:17 smithi145 ceph-mon[89145]: osdmap e705: 8 total, 8 up, 8 in 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3336: done (4 left) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3337: done (3 left) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3338: done (2 left) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3339: done (1 left) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3341: done (0 left) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3342: read oid 37 snap -1 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3342: expect (ObjNum 986 snap 307 seq_num 986) 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3343: rmattr oid 21 current snap is 321 2024-09-13T14:30:18.058 INFO:tasks.rados.rados.0.smithi026.stdout:3344: snap_create 2024-09-13T14:30:18.065 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1654 (ObjNum 65 snap 0 seq_num 1316049258) dirty exists 2024-09-13T14:30:18.067 INFO:tasks.rados.rados.0.smithi026.stdout:3342: expect (ObjNum 986 snap 307 seq_num 986) 2024-09-13T14:30:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:18 smithi026 ceph-mon[97022]: pgmap v664: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-13T14:30:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:18 smithi026 ceph-mon[97022]: osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:30:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:18 smithi026 ceph-mon[99461]: pgmap v664: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-13T14:30:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:18 smithi026 ceph-mon[99461]: osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:30:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:18 smithi145 ceph-mon[89145]: pgmap v664: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-13T14:30:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:18 smithi145 ceph-mon[89145]: osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:30:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3342: done (2 left) 2024-09-13T14:30:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3343: done (1 left) 2024-09-13T14:30:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3344: done (0 left) 2024-09-13T14:30:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3345: read oid 43 snap -1 2024-09-13T14:30:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3345: expect (ObjNum 1004 snap 311 seq_num 1004) 2024-09-13T14:30:19.070 INFO:tasks.rados.rados.0.smithi026.stdout:3346: snap_create 2024-09-13T14:30:19.078 INFO:tasks.rados.rados.0.smithi026.stdout:3345: expect (ObjNum 1004 snap 311 seq_num 1004) 2024-09-13T14:30:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:19 smithi026 ceph-mon[97022]: osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:30:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:19 smithi026 ceph-mon[99461]: osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:30:19.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:19 smithi145 ceph-mon[89145]: osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:30:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:3345: done (1 left) 2024-09-13T14:30:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:3346: done (0 left) 2024-09-13T14:30:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 323 2024-09-13T14:30:20.070 INFO:tasks.rados.rados.0.smithi026.stdout:3347: seq_num 1051 ranges {1540096=409600,1949696=401408} 2024-09-13T14:30:20.080 INFO:tasks.rados.rados.0.smithi026.stdout:3347: writing smithi02683976-7 from 1540096 to 1949696 tid 1 2024-09-13T14:30:20.083 INFO:tasks.rados.rados.0.smithi026.stdout:3347: writing smithi02683976-7 from 1949696 to 2351104 tid 2 2024-09-13T14:30:20.083 INFO:tasks.rados.rados.0.smithi026.stdout:3348: delete oid 47 current snap is 323 2024-09-13T14:30:20.087 INFO:tasks.rados.rados.0.smithi026.stdout:3347: finishing write tid 1 to smithi02683976-7 2024-09-13T14:30:20.087 INFO:tasks.rados.rados.0.smithi026.stdout:3347: finishing write tid 2 to smithi02683976-7 2024-09-13T14:30:20.088 INFO:tasks.rados.rados.0.smithi026.stdout:3348: done (1 left) 2024-09-13T14:30:20.088 INFO:tasks.rados.rados.0.smithi026.stdout:3349: read oid 21 snap -1 2024-09-13T14:30:20.088 INFO:tasks.rados.rados.0.smithi026.stdout:3349: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:20.088 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 323 2024-09-13T14:30:20.088 INFO:tasks.rados.rados.0.smithi026.stdout:3350: seq_num 1053 ranges {704512=442368,1146880=401408} 2024-09-13T14:30:20.095 INFO:tasks.rados.rados.0.smithi026.stdout:3350: writing smithi02683976-33 from 704512 to 1146880 tid 1 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3350: writing smithi02683976-33 from 1146880 to 1548288 tid 2 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3351: rmattr oid 42 current snap is 323 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3351: done (3 left) 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3352: copy_from oid 16 from oid 46 current snap is 323 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3347: finishing write tid 3 to smithi02683976-7 2024-09-13T14:30:20.098 INFO:tasks.rados.rados.0.smithi026.stdout:3349: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:20.100 INFO:tasks.rados.rados.0.smithi026.stdout:3347: finishing write tid 4 to smithi02683976-7 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1608 (ObjNum 1051 snap 323 seq_num 1051) dirty exists 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:3347: left oid 7 (ObjNum 1051 snap 323 seq_num 1051) 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:3347: done (3 left) 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:3349: done (2 left) 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 45 current snap is 323 2024-09-13T14:30:20.101 INFO:tasks.rados.rados.0.smithi026.stdout:3353: seq_num 1054 ranges {794624=458752,1253376=401408} 2024-09-13T14:30:20.108 INFO:tasks.rados.rados.0.smithi026.stdout:3353: writing smithi02683976-45 from 794624 to 1253376 tid 1 2024-09-13T14:30:20.110 INFO:tasks.rados.rados.0.smithi026.stdout:3353: writing smithi02683976-45 from 1253376 to 1654784 tid 2 2024-09-13T14:30:20.110 INFO:tasks.rados.rados.0.smithi026.stdout:3354: setattr oid 22 current snap is 323 2024-09-13T14:30:20.110 INFO:tasks.rados.rados.0.smithi026.stdout:3350: finishing write tid 1 to smithi02683976-33 2024-09-13T14:30:20.110 INFO:tasks.rados.rados.0.smithi026.stdout:3350: finishing write tid 2 to smithi02683976-33 2024-09-13T14:30:20.110 INFO:tasks.rados.rados.0.smithi026.stdout:3350: finishing write tid 3 to smithi02683976-33 2024-09-13T14:30:20.111 INFO:tasks.rados.rados.0.smithi026.stdout:3350: finishing write tid 4 to smithi02683976-33 2024-09-13T14:30:20.111 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2002 (ObjNum 1053 snap 323 seq_num 1053) dirty exists 2024-09-13T14:30:20.111 INFO:tasks.rados.rados.0.smithi026.stdout:3350: left oid 33 (ObjNum 1053 snap 323 seq_num 1053) 2024-09-13T14:30:20.112 INFO:tasks.rados.rados.0.smithi026.stdout:3350: done (3 left) 2024-09-13T14:30:20.112 INFO:tasks.rados.rados.0.smithi026.stdout:3355: read oid 39 snap -1 2024-09-13T14:30:20.112 INFO:tasks.rados.rados.0.smithi026.stdout:3355: expect (ObjNum 972 snap 304 seq_num 972) 2024-09-13T14:30:20.113 INFO:tasks.rados.rados.0.smithi026.stdout:3356: rollback oid 28 current snap is 323 2024-09-13T14:30:20.113 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 320 2024-09-13T14:30:20.113 INFO:tasks.rados.rados.0.smithi026.stdout:3357: setattr oid 44 current snap is 323 2024-09-13T14:30:20.113 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 323 2024-09-13T14:30:20.114 INFO:tasks.rados.rados.0.smithi026.stdout:3358: seq_num 1055 ranges {7888896=475136,8364032=401408} 2024-09-13T14:30:20.160 INFO:tasks.rados.rados.0.smithi026.stdout:3358: writing smithi02683976-19 from 7888896 to 8364032 tid 1 2024-09-13T14:30:20.162 INFO:tasks.rados.rados.0.smithi026.stdout:3358: writing smithi02683976-19 from 8364032 to 8765440 tid 2 2024-09-13T14:30:20.162 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 323 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3352: finishing copy_from racing read to smithi02683976-16 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1617 (ObjNum 33 snap 0 seq_num 520395600) dirty exists 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1639 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3353: finishing write tid 1 to smithi02683976-45 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3353: finishing write tid 2 to smithi02683976-45 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3353: finishing write tid 3 to smithi02683976-45 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3356: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3356: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:30:20.163 INFO:tasks.rados.rados.0.smithi026.stdout:3355: expect (ObjNum 972 snap 304 seq_num 972) 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:3353: finishing write tid 4 to smithi02683976-45 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1204 (ObjNum 1054 snap 323 seq_num 1054) dirty exists 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:3353: left oid 45 (ObjNum 1054 snap 323 seq_num 1054) 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:3356: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1631 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:3352: finishing copy_from to smithi02683976-16 2024-09-13T14:30:20.262 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 1635 (ObjNum 1040 snap 319 seq_num 1040) dirty exists 2024-09-13T14:30:20.263 INFO:tasks.rados.rados.0.smithi026.stdout:3359: seq_num 1056 ranges {3776512=491520,4268032=401408} 2024-09-13T14:30:20.285 INFO:tasks.rados.rados.0.smithi026.stdout:3359: writing smithi02683976-27 from 3776512 to 4268032 tid 1 2024-09-13T14:30:20.287 INFO:tasks.rados.rados.0.smithi026.stdout:3359: writing smithi02683976-27 from 4268032 to 4669440 tid 2 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3352: done (7 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3353: done (6 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3354: done (5 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3355: done (4 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3356: done (3 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3357: done (2 left) 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3360: delete oid 14 current snap is 323 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3358: finishing write tid 1 to smithi02683976-19 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3358: finishing write tid 2 to smithi02683976-19 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3358: finishing write tid 3 to smithi02683976-19 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:3358: finishing write tid 4 to smithi02683976-19 2024-09-13T14:30:20.288 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 1661 (ObjNum 1055 snap 323 seq_num 1055) dirty exists 2024-09-13T14:30:20.289 INFO:tasks.rados.rados.0.smithi026.stdout:3358: left oid 19 (ObjNum 1055 snap 323 seq_num 1055) 2024-09-13T14:30:20.290 INFO:tasks.rados.rados.0.smithi026.stdout:3359: finishing write tid 1 to smithi02683976-27 2024-09-13T14:30:20.290 INFO:tasks.rados.rados.0.smithi026.stdout:3358: done (2 left) 2024-09-13T14:30:20.290 INFO:tasks.rados.rados.0.smithi026.stdout:3360: done (1 left) 2024-09-13T14:30:20.290 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 323 2024-09-13T14:30:20.291 INFO:tasks.rados.rados.0.smithi026.stdout:3361: seq_num 1058 ranges {3473408=524288,3997696=401408} 2024-09-13T14:30:20.312 INFO:tasks.rados.rados.0.smithi026.stdout:3361: writing smithi02683976-10 from 3473408 to 3997696 tid 1 2024-09-13T14:30:20.314 INFO:tasks.rados.rados.0.smithi026.stdout:3361: writing smithi02683976-10 from 3997696 to 4399104 tid 2 2024-09-13T14:30:20.314 INFO:tasks.rados.rados.0.smithi026.stdout:3362: snap_create 2024-09-13T14:30:20.315 INFO:tasks.rados.rados.0.smithi026.stdout:3359: finishing write tid 2 to smithi02683976-27 2024-09-13T14:30:20.315 INFO:tasks.rados.rados.0.smithi026.stdout:3359: finishing write tid 3 to smithi02683976-27 2024-09-13T14:30:20.315 INFO:tasks.rados.rados.0.smithi026.stdout:3359: finishing write tid 4 to smithi02683976-27 2024-09-13T14:30:20.315 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2006 (ObjNum 1056 snap 323 seq_num 1056) dirty exists 2024-09-13T14:30:20.315 INFO:tasks.rados.rados.0.smithi026.stdout:3359: left oid 27 (ObjNum 1056 snap 323 seq_num 1056) 2024-09-13T14:30:20.317 INFO:tasks.rados.rados.0.smithi026.stdout:3361: finishing write tid 1 to smithi02683976-10 2024-09-13T14:30:20.317 INFO:tasks.rados.rados.0.smithi026.stdout:3361: finishing write tid 2 to smithi02683976-10 2024-09-13T14:30:20.317 INFO:tasks.rados.rados.0.smithi026.stdout:3361: finishing write tid 3 to smithi02683976-10 2024-09-13T14:30:20.318 INFO:tasks.rados.rados.0.smithi026.stdout:3361: finishing write tid 4 to smithi02683976-10 2024-09-13T14:30:20.318 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1634 (ObjNum 1058 snap 323 seq_num 1058) dirty exists 2024-09-13T14:30:20.318 INFO:tasks.rados.rados.0.smithi026.stdout:3361: left oid 10 (ObjNum 1058 snap 323 seq_num 1058) 2024-09-13T14:30:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:20 smithi026 ceph-mon[97022]: pgmap v667: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-13T14:30:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:20 smithi026 ceph-mon[97022]: osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:30:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:20 smithi026 ceph-mon[99461]: pgmap v667: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-13T14:30:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:20 smithi026 ceph-mon[99461]: osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:30:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:20 smithi145 ceph-mon[89145]: pgmap v667: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 607 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-13T14:30:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:20 smithi145 ceph-mon[89145]: osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:30:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:21.081 INFO:tasks.rados.rados.0.smithi026.stdout:3359: done (2 left) 2024-09-13T14:30:21.081 INFO:tasks.rados.rados.0.smithi026.stdout:3361: done (1 left) 2024-09-13T14:30:21.081 INFO:tasks.rados.rados.0.smithi026.stdout:3362: done (0 left) 2024-09-13T14:30:21.081 INFO:tasks.rados.rados.0.smithi026.stdout:3363: read oid 2 snap 321 2024-09-13T14:30:21.081 INFO:tasks.rados.rados.0.smithi026.stdout:3363: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:30:21.082 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 12 current snap is 324 2024-09-13T14:30:21.082 INFO:tasks.rados.rados.0.smithi026.stdout:3364: seq_num 1059 ranges {5480448=540672,6021120=401408} 2024-09-13T14:30:21.114 INFO:tasks.rados.rados.0.smithi026.stdout:3364: writing smithi02683976-12 from 5480448 to 6021120 tid 1 2024-09-13T14:30:21.116 INFO:tasks.rados.rados.0.smithi026.stdout:3364: writing smithi02683976-12 from 6021120 to 6422528 tid 2 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3365: read oid 18 snap -1 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3365: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3366: read oid 21 snap -1 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3366: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3367: rollback oid 22 current snap is 324 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 320 2024-09-13T14:30:21.117 INFO:tasks.rados.rados.0.smithi026.stdout:3363: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:30:21.128 INFO:tasks.rados.rados.0.smithi026.stdout:3365: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:21.130 INFO:tasks.rados.rados.0.smithi026.stdout:3366: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3364: finishing write tid 1 to smithi02683976-12 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3364: finishing write tid 2 to smithi02683976-12 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3364: finishing write tid 3 to smithi02683976-12 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3367: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3364: finishing write tid 4 to smithi02683976-12 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1620 (ObjNum 1059 snap 324 seq_num 1059) dirty exists 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3364: left oid 12 (ObjNum 1059 snap 324 seq_num 1059) 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:3367: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:30:21.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 1622 (ObjNum 4113 snap 0 seq_num 4184127801) dirty dne 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3363: done (4 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3364: done (3 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3365: done (2 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3366: done (1 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3367: done (0 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3368: rmattr oid 42 current snap is 324 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3368: done (0 left) 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 324 2024-09-13T14:30:21.133 INFO:tasks.rados.rados.0.smithi026.stdout:3369: seq_num 1060 ranges {2121728=557056,2678784=401408} 2024-09-13T14:30:21.146 INFO:tasks.rados.rados.0.smithi026.stdout:3369: writing smithi02683976-37 from 2121728 to 2678784 tid 1 2024-09-13T14:30:21.149 INFO:tasks.rados.rados.0.smithi026.stdout:3369: writing smithi02683976-37 from 2678784 to 3080192 tid 2 2024-09-13T14:30:21.149 INFO:tasks.rados.rados.0.smithi026.stdout:3370: snap_create 2024-09-13T14:30:21.152 INFO:tasks.rados.rados.0.smithi026.stdout:3369: finishing write tid 1 to smithi02683976-37 2024-09-13T14:30:21.152 INFO:tasks.rados.rados.0.smithi026.stdout:3369: finishing write tid 2 to smithi02683976-37 2024-09-13T14:30:21.152 INFO:tasks.rados.rados.0.smithi026.stdout:3369: finishing write tid 3 to smithi02683976-37 2024-09-13T14:30:21.153 INFO:tasks.rados.rados.0.smithi026.stdout:3369: finishing write tid 4 to smithi02683976-37 2024-09-13T14:30:21.153 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1473 (ObjNum 1060 snap 324 seq_num 1060) dirty exists 2024-09-13T14:30:21.153 INFO:tasks.rados.rados.0.smithi026.stdout:3369: left oid 37 (ObjNum 1060 snap 324 seq_num 1060) 2024-09-13T14:30:21.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:21 smithi026 ceph-mon[97022]: osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:30:21.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:21 smithi026 ceph-mon[99461]: osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:30:21.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:21 smithi145 ceph-mon[89145]: osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:30:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:21] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:30:22.086 INFO:tasks.rados.rados.0.smithi026.stdout:3369: done (1 left) 2024-09-13T14:30:22.086 INFO:tasks.rados.rados.0.smithi026.stdout:3370: done (0 left) 2024-09-13T14:30:22.086 INFO:tasks.rados.rados.0.smithi026.stdout:3371: copy_from oid 19 from oid 27 current snap is 325 2024-09-13T14:30:22.086 INFO:tasks.rados.rados.0.smithi026.stdout:3372: snap_remove snap 306 2024-09-13T14:30:22.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:22 smithi026 ceph-mon[97022]: pgmap v670: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:30:22.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:22 smithi026 ceph-mon[97022]: osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:30:22.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:22 smithi026 ceph-mon[99461]: pgmap v670: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:30:22.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:22 smithi026 ceph-mon[99461]: osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:30:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:22 smithi145 ceph-mon[89145]: pgmap v670: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:30:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:22 smithi145 ceph-mon[89145]: osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:30:23.094 INFO:tasks.rados.rados.0.smithi026.stdout:3372: done (1 left) 2024-09-13T14:30:23.095 INFO:tasks.rados.rados.0.smithi026.stdout:3373: read oid 21 snap 313 2024-09-13T14:30:23.095 INFO:tasks.rados.rados.0.smithi026.stdout:3373: expect (ObjNum 520245520 snap 21981 seq_num 540626304) 2024-09-13T14:30:23.095 INFO:tasks.rados.rados.0.smithi026.stdout:3374: snap_remove snap 320 2024-09-13T14:30:23.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:23 smithi026 ceph-mon[97022]: osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:30:23.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:23 smithi026 ceph-mon[99461]: osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:30:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:23 smithi145 ceph-mon[89145]: osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:30:24.012 INFO:tasks.rados.rados.0.smithi026.stdout:3374: done (2 left) 2024-09-13T14:30:24.013 INFO:tasks.rados.rados.0.smithi026.stdout:3375: snap_remove snap 318 2024-09-13T14:30:24.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[97022]: pgmap v673: 121 pgs: 121 active+clean; 617 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 18 op/s 2024-09-13T14:30:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[97022]: osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:30:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[97022]: osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:30:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[99461]: pgmap v673: 121 pgs: 121 active+clean; 617 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 18 op/s 2024-09-13T14:30:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[99461]: osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:30:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:24 smithi026 ceph-mon[99461]: osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:30:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:24 smithi145 ceph-mon[89145]: pgmap v673: 121 pgs: 121 active+clean; 617 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.7 MiB/s wr, 18 op/s 2024-09-13T14:30:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:24 smithi145 ceph-mon[89145]: osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:30:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:24 smithi145 ceph-mon[89145]: osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:30:25.013 INFO:tasks.rados.rados.0.smithi026.stdout:3375: done (2 left) 2024-09-13T14:30:25.013 INFO:tasks.rados.rados.0.smithi026.stdout:3376: snap_create 2024-09-13T14:30:25.013 INFO:tasks.rados.rados.0.smithi026.stdout:3371: finishing copy_from racing read to smithi02683976-19 2024-09-13T14:30:25.013 INFO:tasks.rados.rados.0.smithi026.stdout:3371: finishing copy_from to smithi02683976-19 2024-09-13T14:30:25.013 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2006 (ObjNum 1056 snap 323 seq_num 1056) dirty exists 2024-09-13T14:30:25.014 INFO:tasks.rados.rados.0.smithi026.stdout:3373: expect (ObjNum 520245520 snap 21981 seq_num 540626304) 2024-09-13T14:30:26.015 INFO:tasks.rados.rados.0.smithi026.stdout:3371: done (2 left) 2024-09-13T14:30:26.015 INFO:tasks.rados.rados.0.smithi026.stdout:3373: done (1 left) 2024-09-13T14:30:26.015 INFO:tasks.rados.rados.0.smithi026.stdout:3376: done (0 left) 2024-09-13T14:30:26.015 INFO:tasks.rados.rados.0.smithi026.stdout:3377: rmattr oid 20 current snap is 326 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3377: done (0 left) 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3378: rollback oid 4 current snap is 326 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 295 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3379: rmattr oid 11 current snap is 326 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3379: done (1 left) 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3380: read oid 23 snap 309 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3380: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 24 current snap is 326 2024-09-13T14:30:26.016 INFO:tasks.rados.rados.0.smithi026.stdout:3381: seq_num 1061 ranges {4276224=573440,4849664=401408} 2024-09-13T14:30:26.042 INFO:tasks.rados.rados.0.smithi026.stdout:3381: writing smithi02683976-24 from 4276224 to 4849664 tid 1 2024-09-13T14:30:26.045 INFO:tasks.rados.rados.0.smithi026.stdout:3381: writing smithi02683976-24 from 4849664 to 5251072 tid 2 2024-09-13T14:30:26.045 INFO:tasks.rados.rados.0.smithi026.stdout:3382: rmattr oid 8 current snap is 326 2024-09-13T14:30:26.045 INFO:tasks.rados.rados.0.smithi026.stdout:3382: done (3 left) 2024-09-13T14:30:26.045 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 326 2024-09-13T14:30:26.046 INFO:tasks.rados.rados.0.smithi026.stdout:3378: finishing rollback tid 0 to smithi02683976-4 2024-09-13T14:30:26.046 INFO:tasks.rados.rados.0.smithi026.stdout:3378: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:30:26.046 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1821 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-13T14:30:26.046 INFO:tasks.rados.rados.0.smithi026.stdout:3383: seq_num 1062 ranges {3694592=589824,4284416=401408} 2024-09-13T14:30:26.069 INFO:tasks.rados.rados.0.smithi026.stdout:3383: writing smithi02683976-49 from 3694592 to 4284416 tid 1 2024-09-13T14:30:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3383: writing smithi02683976-49 from 4284416 to 4685824 tid 2 2024-09-13T14:30:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3378: done (3 left) 2024-09-13T14:30:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 326 2024-09-13T14:30:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3381: finishing write tid 1 to smithi02683976-24 2024-09-13T14:30:26.071 INFO:tasks.rados.rados.0.smithi026.stdout:3381: finishing write tid 2 to smithi02683976-24 2024-09-13T14:30:26.072 INFO:tasks.rados.rados.0.smithi026.stdout:3381: finishing write tid 3 to smithi02683976-24 2024-09-13T14:30:26.072 INFO:tasks.rados.rados.0.smithi026.stdout:3381: finishing write tid 4 to smithi02683976-24 2024-09-13T14:30:26.072 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1317 (ObjNum 1061 snap 326 seq_num 1061) dirty exists 2024-09-13T14:30:26.072 INFO:tasks.rados.rados.0.smithi026.stdout:3381: left oid 24 (ObjNum 1061 snap 326 seq_num 1061) 2024-09-13T14:30:26.072 INFO:tasks.rados.rados.0.smithi026.stdout:3380: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:30:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:26 smithi026 ceph-mon[97022]: pgmap v676: 121 pgs: 121 active+clean; 618 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:30:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:26 smithi026 ceph-mon[97022]: osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:30:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:26 smithi026 ceph-mon[99461]: pgmap v676: 121 pgs: 121 active+clean; 618 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:30:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:26 smithi026 ceph-mon[99461]: osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:30:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:26 smithi145 ceph-mon[89145]: pgmap v676: 121 pgs: 121 active+clean; 618 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-13T14:30:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:26 smithi145 ceph-mon[89145]: osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:30:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:3383: finishing write tid 1 to smithi02683976-49 2024-09-13T14:30:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:3383: finishing write tid 2 to smithi02683976-49 2024-09-13T14:30:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:3383: finishing write tid 3 to smithi02683976-49 2024-09-13T14:30:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:3383: finishing write tid 4 to smithi02683976-49 2024-09-13T14:30:27.256 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 1637 (ObjNum 1062 snap 326 seq_num 1062) dirty exists 2024-09-13T14:30:27.257 INFO:tasks.rados.rados.0.smithi026.stdout:3383: left oid 49 (ObjNum 1062 snap 326 seq_num 1062) 2024-09-13T14:30:27.257 INFO:tasks.rados.rados.0.smithi026.stdout:3384: seq_num 1063 ranges {0=606208,606208=401408} 2024-09-13T14:30:27.259 INFO:tasks.rados.rados.0.smithi026.stdout:3384: writing smithi02683976-47 from 0 to 606208 tid 1 2024-09-13T14:30:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:3384: writing smithi02683976-47 from 606208 to 1007616 tid 2 2024-09-13T14:30:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:3380: done (3 left) 2024-09-13T14:30:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:3381: done (2 left) 2024-09-13T14:30:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:3383: done (1 left) 2024-09-13T14:30:27.262 INFO:tasks.rados.rados.0.smithi026.stdout:3385: snap_remove snap 317 2024-09-13T14:30:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:27 smithi026 ceph-mon[99461]: osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:30:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:27 smithi026 ceph-mon[97022]: osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:30:27.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:27 smithi145 ceph-mon[89145]: osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:30:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:28.032 INFO:tasks.rados.rados.0.smithi026.stdout:3385: done (1 left) 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3386: snap_create 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3384: finishing write tid 1 to smithi02683976-47 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3384: finishing write tid 2 to smithi02683976-47 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3384: finishing write tid 3 to smithi02683976-47 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3384: finishing write tid 4 to smithi02683976-47 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1480 (ObjNum 1063 snap 326 seq_num 1063) dirty exists 2024-09-13T14:30:28.033 INFO:tasks.rados.rados.0.smithi026.stdout:3384: left oid 47 (ObjNum 1063 snap 326 seq_num 1063) 2024-09-13T14:30:28.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:28 smithi026 ceph-mon[99461]: pgmap v679: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-13T14:30:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:28 smithi026 ceph-mon[97022]: pgmap v679: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-13T14:30:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:28 smithi145 ceph-mon[89145]: pgmap v679: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 613 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-13T14:30:29.038 INFO:tasks.rados.rados.0.smithi026.stdout:3384: done (1 left) 2024-09-13T14:30:29.038 INFO:tasks.rados.rados.0.smithi026.stdout:3386: done (0 left) 2024-09-13T14:30:29.038 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 327 2024-09-13T14:30:29.038 INFO:tasks.rados.rados.0.smithi026.stdout:3387: seq_num 1064 ranges {3383296=622592,4005888=401408} 2024-09-13T14:30:29.061 INFO:tasks.rados.rados.0.smithi026.stdout:3387: writing smithi02683976-17 from 3383296 to 4005888 tid 1 2024-09-13T14:30:29.061 INFO:tasks.rados.rados.0.smithi026.stdout:3387: writing smithi02683976-17 from 4005888 to 4407296 tid 2 2024-09-13T14:30:29.062 INFO:tasks.rados.rados.0.smithi026.stdout:3388: read oid 7 snap -1 2024-09-13T14:30:29.062 INFO:tasks.rados.rados.0.smithi026.stdout:3388: expect (ObjNum 1051 snap 323 seq_num 1051) 2024-09-13T14:30:29.062 INFO:tasks.rados.rados.0.smithi026.stdout:3389: snap_create 2024-09-13T14:30:29.065 INFO:tasks.rados.rados.0.smithi026.stdout:3387: finishing write tid 1 to smithi02683976-17 2024-09-13T14:30:29.067 INFO:tasks.rados.rados.0.smithi026.stdout:3387: finishing write tid 2 to smithi02683976-17 2024-09-13T14:30:29.068 INFO:tasks.rados.rados.0.smithi026.stdout:3387: finishing write tid 3 to smithi02683976-17 2024-09-13T14:30:29.069 INFO:tasks.rados.rados.0.smithi026.stdout:3387: finishing write tid 4 to smithi02683976-17 2024-09-13T14:30:29.070 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1229 (ObjNum 1064 snap 327 seq_num 1064) dirty exists 2024-09-13T14:30:29.070 INFO:tasks.rados.rados.0.smithi026.stdout:3387: left oid 17 (ObjNum 1064 snap 327 seq_num 1064) 2024-09-13T14:30:29.070 INFO:tasks.rados.rados.0.smithi026.stdout:3388: expect (ObjNum 1051 snap 323 seq_num 1051) 2024-09-13T14:30:29.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:29 smithi026 ceph-mon[97022]: osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:30:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:29 smithi026 ceph-mon[99461]: osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:30:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:29 smithi145 ceph-mon[89145]: osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3387: done (2 left) 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3388: done (1 left) 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3389: done (0 left) 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3390: rmattr oid 35 current snap is 328 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3390: done (0 left) 2024-09-13T14:30:30.044 INFO:tasks.rados.rados.0.smithi026.stdout:3391: snap_create 2024-09-13T14:30:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:30 smithi026 ceph-mon[97022]: pgmap v681: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:30 smithi026 ceph-mon[97022]: osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:30:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:30 smithi026 ceph-mon[99461]: pgmap v681: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:30 smithi026 ceph-mon[99461]: osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:30:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:30 smithi145 ceph-mon[89145]: pgmap v681: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:30 smithi145 ceph-mon[89145]: osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:30:30.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:31.050 INFO:tasks.rados.rados.0.smithi026.stdout:3391: done (0 left) 2024-09-13T14:30:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:3392: read oid 3 snap -1 2024-09-13T14:30:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:3392: expect (ObjNum 1032 snap 317 seq_num 1032) 2024-09-13T14:30:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 329 2024-09-13T14:30:31.051 INFO:tasks.rados.rados.0.smithi026.stdout:3393: seq_num 1065 ranges {0=638976,638976=401408} 2024-09-13T14:30:31.054 INFO:tasks.rados.rados.0.smithi026.stdout:3393: writing smithi02683976-18 from 0 to 638976 tid 1 2024-09-13T14:30:31.057 INFO:tasks.rados.rados.0.smithi026.stdout:3393: writing smithi02683976-18 from 638976 to 1040384 tid 2 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3394: read oid 47 snap -1 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3394: expect (ObjNum 1063 snap 326 seq_num 1063) 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3395: copy_from oid 41 from oid 46 current snap is 329 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3396: read oid 49 snap -1 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3396: expect (ObjNum 1062 snap 326 seq_num 1062) 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3397: read oid 21 snap -1 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3397: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:31.058 INFO:tasks.rados.rados.0.smithi026.stdout:3398: snap_create 2024-09-13T14:30:31.064 INFO:tasks.rados.rados.0.smithi026.stdout:3397: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:31.067 INFO:tasks.rados.rados.0.smithi026.stdout:3393: finishing write tid 1 to smithi02683976-18 2024-09-13T14:30:31.068 INFO:tasks.rados.rados.0.smithi026.stdout:3393: finishing write tid 2 to smithi02683976-18 2024-09-13T14:30:31.068 INFO:tasks.rados.rados.0.smithi026.stdout:3393: finishing write tid 3 to smithi02683976-18 2024-09-13T14:30:31.069 INFO:tasks.rados.rados.0.smithi026.stdout:3394: expect (ObjNum 1063 snap 326 seq_num 1063) 2024-09-13T14:30:31.108 INFO:tasks.rados.rados.0.smithi026.stdout:3395: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:3395: finishing copy_from to smithi02683976-41 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1635 (ObjNum 1040 snap 319 seq_num 1040) dirty exists 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:3393: finishing write tid 4 to smithi02683976-18 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1662 (ObjNum 1065 snap 329 seq_num 1065) dirty exists 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:3393: left oid 18 (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:30:31.109 INFO:tasks.rados.rados.0.smithi026.stdout:3396: expect (ObjNum 1062 snap 326 seq_num 1062) 2024-09-13T14:30:31.313 INFO:tasks.rados.rados.0.smithi026.stdout:3392: expect (ObjNum 1032 snap 317 seq_num 1032) 2024-09-13T14:30:31.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:31 smithi026 ceph-mon[99461]: osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:30:31.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:31 smithi026 ceph-mon[97022]: osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:30:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:31 smithi145 ceph-mon[89145]: osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:30:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:31] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.51.0" 2024-09-13T14:30:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:32 smithi026 ceph-mon[97022]: pgmap v684: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T14:30:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:32 smithi026 ceph-mon[97022]: osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:30:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:32 smithi026 ceph-mon[99461]: pgmap v684: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T14:30:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:32 smithi026 ceph-mon[99461]: osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:30:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:32 smithi145 ceph-mon[89145]: pgmap v684: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.7 MiB/s wr, 4 op/s 2024-09-13T14:30:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:32 smithi145 ceph-mon[89145]: osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:30:32.891 INFO:tasks.rados.rados.0.smithi026.stdout:3392: done (6 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3393: done (5 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3394: done (4 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3395: done (3 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3396: done (2 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3397: done (1 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3398: done (0 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3399: rmattr oid 26 current snap is 330 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3399: done (0 left) 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3400: rollback oid 28 current snap is 330 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 28 to 301 2024-09-13T14:30:32.892 INFO:tasks.rados.rados.0.smithi026.stdout:3401: read oid 15 snap 300 2024-09-13T14:30:32.893 INFO:tasks.rados.rados.0.smithi026.stdout:3401: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:32.893 INFO:tasks.rados.rados.0.smithi026.stdout:3402: read oid 38 snap -1 2024-09-13T14:30:32.893 INFO:tasks.rados.rados.0.smithi026.stdout:3402: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:32.893 INFO:tasks.rados.rados.0.smithi026.stdout:3403: snap_remove snap 271 2024-09-13T14:30:33.060 INFO:tasks.rados.rados.0.smithi026.stdout:3403: done (3 left) 2024-09-13T14:30:33.060 INFO:tasks.rados.rados.0.smithi026.stdout:3404: delete oid 49 current snap is 330 2024-09-13T14:30:33.060 INFO:tasks.rados.rados.0.smithi026.stdout:3402: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-13T14:30:33.063 INFO:tasks.rados.rados.0.smithi026.stdout:3400: finishing rollback tid 0 to smithi02683976-28 2024-09-13T14:30:33.063 INFO:tasks.rados.rados.0.smithi026.stdout:3400: finishing rollback tid 1 to smithi02683976-28 2024-09-13T14:30:33.064 INFO:tasks.rados.rados.0.smithi026.stdout:3400: finishing rollback tid 2 to smithi02683976-28 2024-09-13T14:30:33.064 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1640 (ObjNum 882 snap 275 seq_num 882) dirty exists 2024-09-13T14:30:33.064 INFO:tasks.rados.rados.0.smithi026.stdout:3401: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:33.238 INFO:tasks.rados.rados.0.smithi026.stdout:3400: done (3 left) 2024-09-13T14:30:33.239 INFO:tasks.rados.rados.0.smithi026.stdout:3401: done (2 left) 2024-09-13T14:30:33.239 INFO:tasks.rados.rados.0.smithi026.stdout:3402: done (1 left) 2024-09-13T14:30:33.239 INFO:tasks.rados.rados.0.smithi026.stdout:3404: done (0 left) 2024-09-13T14:30:33.239 INFO:tasks.rados.rados.0.smithi026.stdout:3405: snap_remove snap 323 2024-09-13T14:30:33.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:33 smithi026 ceph-mon[97022]: osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:30:33.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:33 smithi026 ceph-mon[99461]: osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:30:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:33 smithi145 ceph-mon[89145]: osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:30:34.014 INFO:tasks.rados.rados.0.smithi026.stdout:3405: done (0 left) 2024-09-13T14:30:34.015 INFO:tasks.rados.rados.0.smithi026.stdout:3406: read oid 35 snap 326 2024-09-13T14:30:34.015 INFO:tasks.rados.rados.0.smithi026.stdout:3406: expect deleted 2024-09-13T14:30:34.015 INFO:tasks.rados.rados.0.smithi026.stdout:3407: copy_from oid 34 from oid 5 current snap is 330 2024-09-13T14:30:34.016 INFO:tasks.rados.rados.0.smithi026.stdout:3408: setattr oid 47 current snap is 330 2024-09-13T14:30:34.016 INFO:tasks.rados.rados.0.smithi026.stdout:3409: snap_create 2024-09-13T14:30:34.025 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1481 (ObjNum 1063 snap 326 seq_num 1063) dirty exists 2024-09-13T14:30:34.039 INFO:tasks.rados.rados.0.smithi026.stdout:3407: finishing copy_from racing read to smithi02683976-34 2024-09-13T14:30:34.041 INFO:tasks.rados.rados.0.smithi026.stdout:3407: finishing copy_from to smithi02683976-34 2024-09-13T14:30:34.041 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1926 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:30:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[97022]: pgmap v687: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 738 KiB/s wr, 4 op/s 2024-09-13T14:30:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[97022]: osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:30:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[97022]: osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:30:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[99461]: pgmap v687: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 738 KiB/s wr, 4 op/s 2024-09-13T14:30:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[99461]: osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:30:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:34 smithi026 ceph-mon[99461]: osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:30:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:34 smithi145 ceph-mon[89145]: pgmap v687: 121 pgs: 121 active+clean; 612 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 738 KiB/s wr, 4 op/s 2024-09-13T14:30:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:34 smithi145 ceph-mon[89145]: osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:30:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:34 smithi145 ceph-mon[89145]: osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:30:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:3406: done (3 left) 2024-09-13T14:30:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:3407: done (2 left) 2024-09-13T14:30:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:3408: done (1 left) 2024-09-13T14:30:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:3409: done (0 left) 2024-09-13T14:30:35.015 INFO:tasks.rados.rados.0.smithi026.stdout:3410: delete oid 34 current snap is 331 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3410: done (0 left) 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3411: rmattr oid 6 current snap is 331 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3411: done (0 left) 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3412: read oid 36 snap -1 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3412: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:35.021 INFO:tasks.rados.rados.0.smithi026.stdout:3413: delete oid 50 current snap is 331 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3413: done (1 left) 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3414: rollback oid 25 current snap is 331 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 330 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3415: read oid 20 snap 324 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3415: expect deleted 2024-09-13T14:30:35.024 INFO:tasks.rados.rados.0.smithi026.stdout:3416: snap_remove snap 309 2024-09-13T14:30:36.028 INFO:tasks.rados.rados.0.smithi026.stdout:3416: done (3 left) 2024-09-13T14:30:36.028 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 16 current snap is 331 2024-09-13T14:30:36.029 INFO:tasks.rados.rados.0.smithi026.stdout:3417: seq_num 1069 ranges {3817472=712704,4530176=401408} 2024-09-13T14:30:36.048 INFO:tasks.rados.rados.0.smithi026.stdout:3417: writing smithi02683976-16 from 3817472 to 4530176 tid 1 2024-09-13T14:30:36.051 INFO:tasks.rados.rados.0.smithi026.stdout:3417: writing smithi02683976-16 from 4530176 to 4931584 tid 2 2024-09-13T14:30:36.051 INFO:tasks.rados.rados.0.smithi026.stdout:3418: copy_from oid 41 from oid 44 current snap is 331 2024-09-13T14:30:36.051 INFO:tasks.rados.rados.0.smithi026.stdout:3414: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:30:36.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1314 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-13T14:30:36.051 INFO:tasks.rados.rados.0.smithi026.stdout:3412: expect (ObjNum 864 snap 266 seq_num 864) 2024-09-13T14:30:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:36 smithi145 ceph-mon[89145]: pgmap v690: 121 pgs: 121 active+clean; 612 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-13T14:30:36.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:36 smithi145 ceph-mon[89145]: osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3417: finishing write tid 1 to smithi02683976-16 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3417: finishing write tid 2 to smithi02683976-16 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3412: done (4 left) 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3414: done (3 left) 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3415: done (2 left) 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3419: read oid 33 snap -1 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3419: expect (ObjNum 1053 snap 323 seq_num 1053) 2024-09-13T14:30:36.220 INFO:tasks.rados.rados.0.smithi026.stdout:3420: snap_remove snap 319 2024-09-13T14:30:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:36 smithi026 ceph-mon[97022]: pgmap v690: 121 pgs: 121 active+clean; 612 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-13T14:30:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:36 smithi026 ceph-mon[97022]: osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:30:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:36 smithi026 ceph-mon[99461]: pgmap v690: 121 pgs: 121 active+clean; 612 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-13T14:30:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:36 smithi026 ceph-mon[99461]: osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:30:37.027 INFO:tasks.rados.rados.0.smithi026.stdout:3420: done (3 left) 2024-09-13T14:30:37.027 INFO:tasks.rados.rados.0.smithi026.stdout:3421: snap_remove snap 295 2024-09-13T14:30:37.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:37 smithi026 ceph-mon[99461]: osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:30:37.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:37 smithi026 ceph-mon[97022]: osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:30:37.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:37 smithi145 ceph-mon[89145]: osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3421: done (3 left) 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3422: copy_from oid 25 from oid 27 current snap is 331 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3423: read oid 30 snap -1 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3423: expect deleted 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3424: rmattr oid 11 current snap is 331 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3424: done (5 left) 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3425: delete oid 23 current snap is 331 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3417: finishing write tid 3 to smithi02683976-16 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3417: finishing write tid 4 to smithi02683976-16 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 2009 (ObjNum 1069 snap 331 seq_num 1069) dirty exists 2024-09-13T14:30:38.035 INFO:tasks.rados.rados.0.smithi026.stdout:3417: left oid 16 (ObjNum 1069 snap 331 seq_num 1069) 2024-09-13T14:30:38.036 INFO:tasks.rados.rados.0.smithi026.stdout:3418: finishing copy_from racing read to smithi02683976-41 2024-09-13T14:30:38.036 INFO:tasks.rados.rados.0.smithi026.stdout:3418: finishing copy_from to smithi02683976-41 2024-09-13T14:30:38.036 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1639 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:30:38.036 INFO:tasks.rados.rados.0.smithi026.stdout:3419: expect (ObjNum 1053 snap 323 seq_num 1053) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3422: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3422: finishing copy_from to smithi02683976-25 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2006 (ObjNum 1056 snap 323 seq_num 1056) dirty exists 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3417: done (5 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3418: done (4 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3419: done (3 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3422: done (2 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3423: done (1 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3425: done (0 left) 2024-09-13T14:30:38.102 INFO:tasks.rados.rados.0.smithi026.stdout:3426: delete oid 40 current snap is 331 2024-09-13T14:30:38.103 INFO:tasks.rados.rados.0.smithi026.stdout:3426: done (0 left) 2024-09-13T14:30:38.104 INFO:tasks.rados.rados.0.smithi026.stdout:3427: delete oid 37 current snap is 331 2024-09-13T14:30:38.106 INFO:tasks.rados.rados.0.smithi026.stdout:3427: done (0 left) 2024-09-13T14:30:38.106 INFO:tasks.rados.rados.0.smithi026.stdout:3428: rmattr oid 49 current snap is 331 2024-09-13T14:30:38.106 INFO:tasks.rados.rados.0.smithi026.stdout:3428: done (0 left) 2024-09-13T14:30:38.106 INFO:tasks.rados.rados.0.smithi026.stdout:3429: snap_remove snap 321 2024-09-13T14:30:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:38 smithi026 ceph-mon[97022]: pgmap v693: 121 pgs: 121 active+clean; 593 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-13T14:30:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:38 smithi026 ceph-mon[97022]: osdmap e725: 8 total, 8 up, 8 in 2024-09-13T14:30:38.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:38 smithi026 ceph-mon[99461]: pgmap v693: 121 pgs: 121 active+clean; 593 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-13T14:30:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:38 smithi026 ceph-mon[99461]: osdmap e725: 8 total, 8 up, 8 in 2024-09-13T14:30:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:38 smithi145 ceph-mon[89145]: pgmap v693: 121 pgs: 121 active+clean; 593 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-13T14:30:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:38 smithi145 ceph-mon[89145]: osdmap e725: 8 total, 8 up, 8 in 2024-09-13T14:30:39.017 INFO:tasks.rados.rados.0.smithi026.stdout:3429: done (0 left) 2024-09-13T14:30:39.017 INFO:tasks.rados.rados.0.smithi026.stdout:3430: rmattr oid 50 current snap is 331 2024-09-13T14:30:39.017 INFO:tasks.rados.rados.0.smithi026.stdout:3430: done (0 left) 2024-09-13T14:30:39.017 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 331 2024-09-13T14:30:39.017 INFO:tasks.rados.rados.0.smithi026.stdout:3431: seq_num 1073 ranges {4628480=778240,5406720=401408} 2024-09-13T14:30:39.046 INFO:tasks.rados.rados.0.smithi026.stdout:3431: writing smithi02683976-44 from 4628480 to 5406720 tid 1 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:3431: writing smithi02683976-44 from 5406720 to 5808128 tid 2 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:3432: rollback oid 4 current snap is 331 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 256 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:3433: read oid 2 snap -1 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:3433: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:30:39.049 INFO:tasks.rados.rados.0.smithi026.stdout:3434: delete oid 40 current snap is 331 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:3432: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:3434: done (3 left) 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:3435: rollback oid 3 current snap is 331 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 308 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 331 2024-09-13T14:30:39.052 INFO:tasks.rados.rados.0.smithi026.stdout:3436: seq_num 1075 ranges {0=409600,409600=671744,1081344=131072} 2024-09-13T14:30:39.055 INFO:tasks.rados.rados.0.smithi026.stdout:3436: writing smithi02683976-9 from 0 to 409600 tid 1 2024-09-13T14:30:39.059 INFO:tasks.rados.rados.0.smithi026.stdout:3436: writing smithi02683976-9 from 409600 to 1081344 tid 2 2024-09-13T14:30:39.060 INFO:tasks.rados.rados.0.smithi026.stdout:3436: writing smithi02683976-9 from 1081344 to 1212416 tid 3 2024-09-13T14:30:39.060 INFO:tasks.rados.rados.0.smithi026.stdout:3437: copy_from oid 12 from oid 33 current snap is 331 2024-09-13T14:30:39.060 INFO:tasks.rados.rados.0.smithi026.stdout:3432: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:30:39.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1823 (ObjNum 781 snap 242 seq_num 781) dirty exists 2024-09-13T14:30:39.061 INFO:tasks.rados.rados.0.smithi026.stdout:3431: finishing write tid 1 to smithi02683976-44 2024-09-13T14:30:39.061 INFO:tasks.rados.rados.0.smithi026.stdout:3431: finishing write tid 2 to smithi02683976-44 2024-09-13T14:30:39.061 INFO:tasks.rados.rados.0.smithi026.stdout:3431: finishing write tid 3 to smithi02683976-44 2024-09-13T14:30:39.061 INFO:tasks.rados.rados.0.smithi026.stdout:3433: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:3431: finishing write tid 4 to smithi02683976-44 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1642 (ObjNum 1073 snap 331 seq_num 1073) dirty exists 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:3431: left oid 44 (ObjNum 1073 snap 331 seq_num 1073) 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:3436: finishing write tid 1 to smithi02683976-9 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:3431: done (5 left) 2024-09-13T14:30:39.074 INFO:tasks.rados.rados.0.smithi026.stdout:3432: done (4 left) 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3433: done (3 left) 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3438: rmattr oid 10 current snap is 331 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3435: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3435: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3435: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 2076 (ObjNum 951 snap 296 seq_num 951) dirty exists 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3436: finishing write tid 2 to smithi02683976-9 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3436: finishing write tid 3 to smithi02683976-9 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3436: finishing write tid 4 to smithi02683976-9 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3436: finishing write tid 5 to smithi02683976-9 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1695 (ObjNum 1075 snap 331 seq_num 1075) dirty exists 2024-09-13T14:30:39.075 INFO:tasks.rados.rados.0.smithi026.stdout:3436: left oid 9 (ObjNum 1075 snap 331 seq_num 1075) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3437: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3435: done (3 left) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3436: done (2 left) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3439: rmattr oid 19 current snap is 331 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3439: done (2 left) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3440: read oid 17 snap -1 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3440: expect (ObjNum 1064 snap 327 seq_num 1064) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3441: rollback oid 22 current snap is 331 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3437: finishing copy_from to smithi02683976-12 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 2002 (ObjNum 1053 snap 323 seq_num 1053) dirty exists 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 304 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3437: done (3 left) 2024-09-13T14:30:39.076 INFO:tasks.rados.rados.0.smithi026.stdout:3442: read oid 14 snap -1 2024-09-13T14:30:39.077 INFO:tasks.rados.rados.0.smithi026.stdout:3442: expect deleted 2024-09-13T14:30:39.077 INFO:tasks.rados.rados.0.smithi026.stdout:3443: delete oid 26 current snap is 331 2024-09-13T14:30:39.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1643 (ObjNum 1058 snap 323 seq_num 1058) dirty exists 2024-09-13T14:30:39.078 INFO:tasks.rados.rados.0.smithi026.stdout:3441: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:30:39.078 INFO:tasks.rados.rados.0.smithi026.stdout:3441: finishing rollback tid 2 to smithi02683976-22 2024-09-13T14:30:39.078 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2004 (ObjNum 884 snap 275 seq_num 884) dirty exists 2024-09-13T14:30:39.078 INFO:tasks.rados.rados.0.smithi026.stdout:3438: done (4 left) 2024-09-13T14:30:39.079 INFO:tasks.rados.rados.0.smithi026.stdout:3441: done (3 left) 2024-09-13T14:30:39.079 INFO:tasks.rados.rados.0.smithi026.stdout:3442: done (2 left) 2024-09-13T14:30:39.079 INFO:tasks.rados.rados.0.smithi026.stdout:3443: done (1 left) 2024-09-13T14:30:39.079 INFO:tasks.rados.rados.0.smithi026.stdout:3444: snap_remove snap 304 2024-09-13T14:30:39.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:39 smithi026 ceph-mon[99461]: osdmap e726: 8 total, 8 up, 8 in 2024-09-13T14:30:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:39 smithi026 ceph-mon[99461]: osdmap e727: 8 total, 8 up, 8 in 2024-09-13T14:30:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:39 smithi026 ceph-mon[97022]: osdmap e726: 8 total, 8 up, 8 in 2024-09-13T14:30:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:39 smithi026 ceph-mon[97022]: osdmap e727: 8 total, 8 up, 8 in 2024-09-13T14:30:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:39 smithi145 ceph-mon[89145]: osdmap e726: 8 total, 8 up, 8 in 2024-09-13T14:30:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:39 smithi145 ceph-mon[89145]: osdmap e727: 8 total, 8 up, 8 in 2024-09-13T14:30:40.047 INFO:tasks.rados.rados.0.smithi026.stdout:3444: done (1 left) 2024-09-13T14:30:40.047 INFO:tasks.rados.rados.0.smithi026.stdout:3445: read oid 10 snap -1 2024-09-13T14:30:40.048 INFO:tasks.rados.rados.0.smithi026.stdout:3445: expect (ObjNum 1058 snap 323 seq_num 1058) 2024-09-13T14:30:40.048 INFO:tasks.rados.rados.0.smithi026.stdout:3446: delete oid 39 current snap is 331 2024-09-13T14:30:40.048 INFO:tasks.rados.rados.0.smithi026.stdout:3440: expect (ObjNum 1064 snap 327 seq_num 1064) 2024-09-13T14:30:40.307 INFO:tasks.rados.rados.0.smithi026.stdout:3445: expect (ObjNum 1058 snap 323 seq_num 1058) 2024-09-13T14:30:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:40 smithi026 ceph-mon[97022]: pgmap v696: 121 pgs: 121 active+clean; 589 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-13T14:30:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:40 smithi026 ceph-mon[99461]: pgmap v696: 121 pgs: 121 active+clean; 589 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-13T14:30:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:40 smithi145 ceph-mon[89145]: pgmap v696: 121 pgs: 121 active+clean; 589 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-13T14:30:40.601 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:40.603 INFO:tasks.rados.rados.0.smithi026.stdout:3440: done (2 left) 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3445: done (1 left) 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3446: done (0 left) 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 331 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3447: seq_num 1078 ranges {0=458752,458752=720896,1179648=81920} 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3447: writing smithi02683976-30 from 0 to 458752 tid 1 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3447: writing smithi02683976-30 from 458752 to 1179648 tid 2 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3447: writing smithi02683976-30 from 1179648 to 1261568 tid 3 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3448: read oid 46 snap -1 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3448: expect (ObjNum 1040 snap 319 seq_num 1040) 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 331 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3447: finishing write tid 1 to smithi02683976-30 2024-09-13T14:30:40.604 INFO:tasks.rados.rados.0.smithi026.stdout:3449: seq_num 1079 ranges {376832=475136,851968=606208,1458176=196608} 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3449: writing smithi02683976-28 from 376832 to 851968 tid 1 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3449: writing smithi02683976-28 from 851968 to 1458176 tid 2 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3449: writing smithi02683976-28 from 1458176 to 1654784 tid 3 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3450: delete oid 44 current snap is 331 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3447: finishing write tid 2 to smithi02683976-30 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3447: finishing write tid 3 to smithi02683976-30 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3447: finishing write tid 4 to smithi02683976-30 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3447: finishing write tid 5 to smithi02683976-30 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1827 (ObjNum 1078 snap 331 seq_num 1078) dirty exists 2024-09-13T14:30:40.605 INFO:tasks.rados.rados.0.smithi026.stdout:3447: left oid 30 (ObjNum 1078 snap 331 seq_num 1078) 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: finishing write tid 1 to smithi02683976-28 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: finishing write tid 2 to smithi02683976-28 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: finishing write tid 3 to smithi02683976-28 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: finishing write tid 4 to smithi02683976-28 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: finishing write tid 5 to smithi02683976-28 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 1647 (ObjNum 1079 snap 331 seq_num 1079) dirty exists 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3449: left oid 28 (ObjNum 1079 snap 331 seq_num 1079) 2024-09-13T14:30:40.606 INFO:tasks.rados.rados.0.smithi026.stdout:3448: expect (ObjNum 1040 snap 319 seq_num 1040) 2024-09-13T14:30:40.708 INFO:tasks.rados.rados.0.smithi026.stdout:3447: done (3 left) 2024-09-13T14:30:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:3448: done (2 left) 2024-09-13T14:30:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:3449: done (1 left) 2024-09-13T14:30:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:3450: done (0 left) 2024-09-13T14:30:40.709 INFO:tasks.rados.rados.0.smithi026.stdout:3451: snap_create 2024-09-13T14:30:41.051 INFO:tasks.rados.rados.0.smithi026.stdout:3451: done (0 left) 2024-09-13T14:30:41.051 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 332 2024-09-13T14:30:41.051 INFO:tasks.rados.rados.0.smithi026.stdout:3452: seq_num 1081 ranges {1196032=507904,1703936=425984,2129920=376832} 2024-09-13T14:30:41.060 INFO:tasks.rados.rados.0.smithi026.stdout:3452: writing smithi02683976-1 from 1196032 to 1703936 tid 1 2024-09-13T14:30:41.063 INFO:tasks.rados.rados.0.smithi026.stdout:3452: writing smithi02683976-1 from 1703936 to 2129920 tid 2 2024-09-13T14:30:41.065 INFO:tasks.rados.rados.0.smithi026.stdout:3452: writing smithi02683976-1 from 2129920 to 2506752 tid 3 2024-09-13T14:30:41.066 INFO:tasks.rados.rados.0.smithi026.stdout:3453: delete oid 50 current snap is 332 2024-09-13T14:30:41.066 INFO:tasks.rados.rados.0.smithi026.stdout:3452: finishing write tid 1 to smithi02683976-1 2024-09-13T14:30:41.069 INFO:tasks.rados.rados.0.smithi026.stdout:3452: finishing write tid 2 to smithi02683976-1 2024-09-13T14:30:41.069 INFO:tasks.rados.rados.0.smithi026.stdout:3452: finishing write tid 3 to smithi02683976-1 2024-09-13T14:30:41.069 INFO:tasks.rados.rados.0.smithi026.stdout:3452: finishing write tid 4 to smithi02683976-1 2024-09-13T14:30:41.070 INFO:tasks.rados.rados.0.smithi026.stdout:3453: done (1 left) 2024-09-13T14:30:41.071 INFO:tasks.rados.rados.0.smithi026.stdout:3454: snap_create 2024-09-13T14:30:41.071 INFO:tasks.rados.rados.0.smithi026.stdout:3452: finishing write tid 5 to smithi02683976-1 2024-09-13T14:30:41.071 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 2093 (ObjNum 1081 snap 332 seq_num 1081) dirty exists 2024-09-13T14:30:41.071 INFO:tasks.rados.rados.0.smithi026.stdout:3452: left oid 1 (ObjNum 1081 snap 332 seq_num 1081) 2024-09-13T14:30:41.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:41 smithi026 ceph-mon[99461]: osdmap e728: 8 total, 8 up, 8 in 2024-09-13T14:30:41.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:41 smithi026 ceph-mon[97022]: osdmap e728: 8 total, 8 up, 8 in 2024-09-13T14:30:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:41 smithi145 ceph-mon[89145]: osdmap e728: 8 total, 8 up, 8 in 2024-09-13T14:30:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:41] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3452: done (1 left) 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3454: done (0 left) 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3455: rmattr oid 15 current snap is 333 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3456: read oid 31 snap -1 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3456: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:30:42.057 INFO:tasks.rados.rados.0.smithi026.stdout:3457: read oid 39 snap -1 2024-09-13T14:30:42.058 INFO:tasks.rados.rados.0.smithi026.stdout:3457: expect deleted 2024-09-13T14:30:42.058 INFO:tasks.rados.rados.0.smithi026.stdout:3458: snap_remove snap 325 2024-09-13T14:30:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[97022]: pgmap v699: 121 pgs: 121 active+clean; 600 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:30:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[97022]: osdmap e729: 8 total, 8 up, 8 in 2024-09-13T14:30:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[99461]: pgmap v699: 121 pgs: 121 active+clean; 600 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:30:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[99461]: osdmap e729: 8 total, 8 up, 8 in 2024-09-13T14:30:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:42 smithi145 ceph-mon[89145]: pgmap v699: 121 pgs: 121 active+clean; 600 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-13T14:30:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:42 smithi145 ceph-mon[89145]: osdmap e729: 8 total, 8 up, 8 in 2024-09-13T14:30:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3458: done (3 left) 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3459: read oid 13 snap 316 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3459: expect deleted 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3460: rollback oid 33 current snap is 333 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 33 to 326 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3461: delete oid 43 current snap is 333 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1828 (ObjNum 931 snap 290 seq_num 931) dirty exists 2024-09-13T14:30:43.065 INFO:tasks.rados.rados.0.smithi026.stdout:3456: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:3460: finishing rollback tid 0 to smithi02683976-33 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:3460: finishing rollback tid 1 to smithi02683976-33 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:3460: finishing rollback tid 2 to smithi02683976-33 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2097 (ObjNum 1053 snap 323 seq_num 1053) dirty exists 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:3455: done (5 left) 2024-09-13T14:30:43.120 INFO:tasks.rados.rados.0.smithi026.stdout:3456: done (4 left) 2024-09-13T14:30:43.121 INFO:tasks.rados.rados.0.smithi026.stdout:3457: done (3 left) 2024-09-13T14:30:43.121 INFO:tasks.rados.rados.0.smithi026.stdout:3459: done (2 left) 2024-09-13T14:30:43.121 INFO:tasks.rados.rados.0.smithi026.stdout:3460: done (1 left) 2024-09-13T14:30:43.121 INFO:tasks.rados.rados.0.smithi026.stdout:3461: done (0 left) 2024-09-13T14:30:43.121 INFO:tasks.rados.rados.0.smithi026.stdout:3462: delete oid 10 current snap is 333 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:3462: done (0 left) 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:3463: rollback oid 11 current snap is 333 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 327 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:3464: rollback oid 20 current snap is 333 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 313 2024-09-13T14:30:43.122 INFO:tasks.rados.rados.0.smithi026.stdout:3465: rollback oid 50 current snap is 333 2024-09-13T14:30:43.123 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 50 to 314 2024-09-13T14:30:43.123 INFO:tasks.rados.rados.0.smithi026.stdout:3466: delete oid 14 current snap is 333 2024-09-13T14:30:43.123 INFO:tasks.rados.rados.0.smithi026.stdout:3464: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:30:43.123 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 0 (ObjNum 145 snap 0 seq_num 519946800) dirty dne 2024-09-13T14:30:43.124 INFO:tasks.rados.rados.0.smithi026.stdout:3464: done (3 left) 2024-09-13T14:30:43.124 INFO:tasks.rados.rados.0.smithi026.stdout:3466: done (2 left) 2024-09-13T14:30:43.125 INFO:tasks.rados.rados.0.smithi026.stdout:3467: delete oid 32 current snap is 333 2024-09-13T14:30:43.126 INFO:tasks.rados.rados.0.smithi026.stdout:3463: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:30:43.126 INFO:tasks.rados.rados.0.smithi026.stdout:3463: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:30:43.127 INFO:tasks.rados.rados.0.smithi026.stdout:3467: done (2 left) 2024-09-13T14:30:43.127 INFO:tasks.rados.rados.0.smithi026.stdout:3468: read oid 31 snap 301 2024-09-13T14:30:43.127 INFO:tasks.rados.rados.0.smithi026.stdout:3468: expect deleted 2024-09-13T14:30:43.127 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 333 2024-09-13T14:30:43.127 INFO:tasks.rados.rados.0.smithi026.stdout:3469: seq_num 1087 ranges {2383872=614400,2998272=532480,3530752=270336} 2024-09-13T14:30:43.143 INFO:tasks.rados.rados.0.smithi026.stdout:3469: writing smithi02683976-36 from 2383872 to 2998272 tid 1 2024-09-13T14:30:43.146 INFO:tasks.rados.rados.0.smithi026.stdout:3469: writing smithi02683976-36 from 2998272 to 3530752 tid 2 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3469: writing smithi02683976-36 from 3530752 to 3801088 tid 3 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3470: copy_from oid 1 from oid 35 current snap is 333 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3463: finishing rollback tid 2 to smithi02683976-11 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1710 (ObjNum 1048 snap 320 seq_num 1048) dirty exists 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3463: done (4 left) 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3471: snap_create 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:3465: finishing rollback tid 1 to smithi02683976-50 2024-09-13T14:30:43.148 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 0 (ObjNum 161 snap 0 seq_num 518813936) dirty dne 2024-09-13T14:30:43.149 INFO:tasks.rados.rados.0.smithi026.stdout:3469: finishing write tid 1 to smithi02683976-36 2024-09-13T14:30:43.150 INFO:tasks.rados.rados.0.smithi026.stdout:3469: finishing write tid 2 to smithi02683976-36 2024-09-13T14:30:43.151 INFO:tasks.rados.rados.0.smithi026.stdout:3469: finishing write tid 3 to smithi02683976-36 2024-09-13T14:30:43.152 INFO:tasks.rados.rados.0.smithi026.stdout:3469: finishing write tid 4 to smithi02683976-36 2024-09-13T14:30:43.153 INFO:tasks.rados.rados.0.smithi026.stdout:3470: finishing copy_from to smithi02683976-1 2024-09-13T14:30:43.153 INFO:tasks.rados.rados.0.smithi026.stdout:3470: got expected ENOENT (src dne) 2024-09-13T14:30:43.153 INFO:tasks.rados.rados.0.smithi026.stdout:3470: finishing copy_from racing read to smithi02683976-1 2024-09-13T14:30:43.154 INFO:tasks.rados.rados.0.smithi026.stdout:3469: finishing write tid 5 to smithi02683976-36 2024-09-13T14:30:43.154 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2106 (ObjNum 1087 snap 333 seq_num 1087) dirty exists 2024-09-13T14:30:43.154 INFO:tasks.rados.rados.0.smithi026.stdout:3469: left oid 36 (ObjNum 1087 snap 333 seq_num 1087) 2024-09-13T14:30:43.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:43 smithi026 ceph-mon[99461]: osdmap e730: 8 total, 8 up, 8 in 2024-09-13T14:30:43.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:43 smithi026 ceph-mon[97022]: osdmap e730: 8 total, 8 up, 8 in 2024-09-13T14:30:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:43 smithi145 ceph-mon[89145]: osdmap e730: 8 total, 8 up, 8 in 2024-09-13T14:30:44.018 INFO:tasks.rados.rados.0.smithi026.stdout:3465: done (4 left) 2024-09-13T14:30:44.018 INFO:tasks.rados.rados.0.smithi026.stdout:3468: done (3 left) 2024-09-13T14:30:44.018 INFO:tasks.rados.rados.0.smithi026.stdout:3469: done (2 left) 2024-09-13T14:30:44.018 INFO:tasks.rados.rados.0.smithi026.stdout:3470: done (1 left) 2024-09-13T14:30:44.019 INFO:tasks.rados.rados.0.smithi026.stdout:3471: done (0 left) 2024-09-13T14:30:44.019 INFO:tasks.rados.rados.0.smithi026.stdout:3472: delete oid 26 current snap is 334 2024-09-13T14:30:44.022 INFO:tasks.rados.rados.0.smithi026.stdout:3472: done (0 left) 2024-09-13T14:30:44.022 INFO:tasks.rados.rados.0.smithi026.stdout:3473: snap_remove snap 307 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[99461]: pgmap v702: 121 pgs: 121 active+clean; 603 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 8 op/s 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[99461]: osdmap e731: 8 total, 8 up, 8 in 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[99461]: osdmap e732: 8 total, 8 up, 8 in 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[97022]: pgmap v702: 121 pgs: 121 active+clean; 603 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 8 op/s 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[97022]: osdmap e731: 8 total, 8 up, 8 in 2024-09-13T14:30:44.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:44 smithi026 ceph-mon[97022]: osdmap e732: 8 total, 8 up, 8 in 2024-09-13T14:30:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:44 smithi145 ceph-mon[89145]: pgmap v702: 121 pgs: 121 active+clean; 603 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.9 MiB/s wr, 8 op/s 2024-09-13T14:30:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:44 smithi145 ceph-mon[89145]: osdmap e731: 8 total, 8 up, 8 in 2024-09-13T14:30:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:44 smithi145 ceph-mon[89145]: osdmap e732: 8 total, 8 up, 8 in 2024-09-13T14:30:45.021 INFO:tasks.rados.rados.0.smithi026.stdout:3473: done (0 left) 2024-09-13T14:30:45.021 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 1 current snap is 334 2024-09-13T14:30:45.021 INFO:tasks.rados.rados.0.smithi026.stdout:3474: seq_num 1089 ranges {2506752=647168,3153920=745472,3899392=57344} 2024-09-13T14:30:45.040 INFO:tasks.rados.rados.0.smithi026.stdout:3474: writing smithi02683976-1 from 2506752 to 3153920 tid 1 2024-09-13T14:30:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3474: writing smithi02683976-1 from 3153920 to 3899392 tid 2 2024-09-13T14:30:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:3474: writing smithi02683976-1 from 3899392 to 3956736 tid 3 2024-09-13T14:30:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 334 2024-09-13T14:30:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:3475: seq_num 1090 ranges {1736704=663552,2400256=630784,3031040=172032} 2024-09-13T14:30:45.059 INFO:tasks.rados.rados.0.smithi026.stdout:3475: writing smithi02683976-31 from 1736704 to 2400256 tid 1 2024-09-13T14:30:45.063 INFO:tasks.rados.rados.0.smithi026.stdout:3475: writing smithi02683976-31 from 2400256 to 3031040 tid 2 2024-09-13T14:30:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3475: writing smithi02683976-31 from 3031040 to 3203072 tid 3 2024-09-13T14:30:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 38 current snap is 334 2024-09-13T14:30:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3474: finishing write tid 1 to smithi02683976-1 2024-09-13T14:30:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3474: finishing write tid 2 to smithi02683976-1 2024-09-13T14:30:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3474: finishing write tid 3 to smithi02683976-1 2024-09-13T14:30:45.066 INFO:tasks.rados.rados.0.smithi026.stdout:3474: finishing write tid 4 to smithi02683976-1 2024-09-13T14:30:45.066 INFO:tasks.rados.rados.0.smithi026.stdout:3476: seq_num 1091 ranges {0=679936,679936=516096,1196032=286720} 2024-09-13T14:30:45.069 INFO:tasks.rados.rados.0.smithi026.stdout:3476: writing smithi02683976-38 from 0 to 679936 tid 1 2024-09-13T14:30:45.072 INFO:tasks.rados.rados.0.smithi026.stdout:3476: writing smithi02683976-38 from 679936 to 1196032 tid 2 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:3476: writing smithi02683976-38 from 1196032 to 1482752 tid 3 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 334 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:3474: finishing write tid 5 to smithi02683976-1 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 2111 (ObjNum 1089 snap 334 seq_num 1089) dirty exists 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:3474: left oid 1 (ObjNum 1089 snap 334 seq_num 1089) 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:3475: finishing write tid 1 to smithi02683976-31 2024-09-13T14:30:45.075 INFO:tasks.rados.rados.0.smithi026.stdout:3477: seq_num 1092 ranges {0=696320,696320=802816} 2024-09-13T14:30:45.079 INFO:tasks.rados.rados.0.smithi026.stdout:3477: writing smithi02683976-44 from 0 to 696320 tid 1 2024-09-13T14:30:45.084 INFO:tasks.rados.rados.0.smithi026.stdout:3477: writing smithi02683976-44 from 696320 to 1499136 tid 2 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3474: done (3 left) 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 334 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3475: finishing write tid 2 to smithi02683976-31 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3475: finishing write tid 3 to smithi02683976-31 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3475: finishing write tid 4 to smithi02683976-31 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3475: finishing write tid 5 to smithi02683976-31 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1726 (ObjNum 1090 snap 334 seq_num 1090) dirty exists 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3475: left oid 31 (ObjNum 1090 snap 334 seq_num 1090) 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3476: finishing write tid 1 to smithi02683976-38 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3476: finishing write tid 2 to smithi02683976-38 2024-09-13T14:30:45.085 INFO:tasks.rados.rados.0.smithi026.stdout:3476: finishing write tid 3 to smithi02683976-38 2024-09-13T14:30:45.086 INFO:tasks.rados.rados.0.smithi026.stdout:3476: finishing write tid 4 to smithi02683976-38 2024-09-13T14:30:45.086 INFO:tasks.rados.rados.0.smithi026.stdout:3478: seq_num 1093 ranges {0=712704,712704=688128,1400832=114688} 2024-09-13T14:30:45.088 INFO:tasks.rados.rados.0.smithi026.stdout:3478: writing smithi02683976-29 from 0 to 712704 tid 1 2024-09-13T14:30:45.092 INFO:tasks.rados.rados.0.smithi026.stdout:3478: writing smithi02683976-29 from 712704 to 1400832 tid 2 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3478: writing smithi02683976-29 from 1400832 to 1515520 tid 3 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3475: done (3 left) 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3479: rmattr oid 19 current snap is 334 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3479: done (3 left) 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3480: rollback oid 27 current snap is 334 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 308 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3476: finishing write tid 5 to smithi02683976-38 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1731 (ObjNum 1091 snap 334 seq_num 1091) dirty exists 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3476: left oid 38 (ObjNum 1091 snap 334 seq_num 1091) 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3477: finishing write tid 1 to smithi02683976-44 2024-09-13T14:30:45.095 INFO:tasks.rados.rados.0.smithi026.stdout:3476: done (3 left) 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3481: rmattr oid 2 current snap is 334 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3481: done (3 left) 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3482: rmattr oid 41 current snap is 334 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3477: finishing write tid 2 to smithi02683976-44 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3477: finishing write tid 3 to smithi02683976-44 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3477: finishing write tid 4 to smithi02683976-44 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1646 (ObjNum 1092 snap 334 seq_num 1092) dirty exists 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3477: left oid 44 (ObjNum 1092 snap 334 seq_num 1092) 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3477: done (3 left) 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3483: read oid 23 snap -1 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3483: expect deleted 2024-09-13T14:30:45.096 INFO:tasks.rados.rados.0.smithi026.stdout:3484: rollback oid 36 current snap is 334 2024-09-13T14:30:45.097 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 314 2024-09-13T14:30:45.097 INFO:tasks.rados.rados.0.smithi026.stdout:3485: read oid 32 snap -1 2024-09-13T14:30:45.097 INFO:tasks.rados.rados.0.smithi026.stdout:3485: expect deleted 2024-09-13T14:30:45.097 INFO:tasks.rados.rados.0.smithi026.stdout:3486: setattr oid 26 current snap is 334 2024-09-13T14:30:45.097 INFO:tasks.rados.rados.0.smithi026.stdout:3478: finishing write tid 1 to smithi02683976-29 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3483: done (6 left) 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3485: done (5 left) 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3487: rmattr oid 30 current snap is 334 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3487: done (5 left) 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3488: read oid 48 snap -1 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3488: expect (ObjNum 1050 snap 320 seq_num 1050) 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3489: read oid 4 snap -1 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3489: expect (ObjNum 781 snap 242 seq_num 781) 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3490: copy_from oid 44 from oid 46 current snap is 334 2024-09-13T14:30:45.099 INFO:tasks.rados.rados.0.smithi026.stdout:3491: rollback oid 38 current snap is 334 2024-09-13T14:30:45.100 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 291 2024-09-13T14:30:45.100 INFO:tasks.rados.rados.0.smithi026.stdout:3492: rollback oid 3 current snap is 334 2024-09-13T14:30:45.100 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 3 to 308 2024-09-13T14:30:45.100 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 334 2024-09-13T14:30:45.100 INFO:tasks.rados.rados.0.smithi026.stdout:3493: seq_num 1094 ranges {4669440=729088,5398528=573440,5971968=229376} 2024-09-13T14:30:45.127 INFO:tasks.rados.rados.0.smithi026.stdout:3493: writing smithi02683976-25 from 4669440 to 5398528 tid 1 2024-09-13T14:30:45.130 INFO:tasks.rados.rados.0.smithi026.stdout:3493: writing smithi02683976-25 from 5398528 to 5971968 tid 2 2024-09-13T14:30:45.132 INFO:tasks.rados.rados.0.smithi026.stdout:3493: writing smithi02683976-25 from 5971968 to 6201344 tid 3 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 334 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1928 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3478: finishing write tid 2 to smithi02683976-29 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3478: finishing write tid 3 to smithi02683976-29 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 2005 (ObjNum 145 snap 0 seq_num 538886704) dirty exists 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3478: finishing write tid 4 to smithi02683976-29 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3478: finishing write tid 5 to smithi02683976-29 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 2014 (ObjNum 1093 snap 334 seq_num 1093) dirty exists 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3478: left oid 29 (ObjNum 1093 snap 334 seq_num 1093) 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3491: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3491: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:30:45.133 INFO:tasks.rados.rados.0.smithi026.stdout:3494: seq_num 1095 ranges {0=745472,745472=499712,1245184=303104} 2024-09-13T14:30:45.136 INFO:tasks.rados.rados.0.smithi026.stdout:3494: writing smithi02683976-23 from 0 to 745472 tid 1 2024-09-13T14:30:45.139 INFO:tasks.rados.rados.0.smithi026.stdout:3494: writing smithi02683976-23 from 745472 to 1245184 tid 2 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3494: writing smithi02683976-23 from 1245184 to 1548288 tid 3 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3478: done (11 left) 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3482: done (10 left) 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3486: done (9 left) 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 334 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3491: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1734 (ObjNum 673 snap 189 seq_num 673) dirty exists 2024-09-13T14:30:45.142 INFO:tasks.rados.rados.0.smithi026.stdout:3488: expect (ObjNum 1050 snap 320 seq_num 1050) 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3480: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3480: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3480: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2125 (ObjNum 820 snap 257 seq_num 820) dirty exists 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3484: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3484: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3484: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2129 (ObjNum 864 snap 266 seq_num 864) dirty exists 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3492: finishing rollback tid 0 to smithi02683976-3 2024-09-13T14:30:45.164 INFO:tasks.rados.rados.0.smithi026.stdout:3492: finishing rollback tid 1 to smithi02683976-3 2024-09-13T14:30:45.165 INFO:tasks.rados.rados.0.smithi026.stdout:3492: finishing rollback tid 2 to smithi02683976-3 2024-09-13T14:30:45.165 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 2133 (ObjNum 951 snap 296 seq_num 951) dirty exists 2024-09-13T14:30:45.165 INFO:tasks.rados.rados.0.smithi026.stdout:3490: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:30:45.165 INFO:tasks.rados.rados.0.smithi026.stdout:3495: seq_num 1096 ranges {3203072=761856,3964928=786432,4751360=16384} 2024-09-13T14:30:45.185 INFO:tasks.rados.rados.0.smithi026.stdout:3495: writing smithi02683976-31 from 3203072 to 3964928 tid 1 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3495: writing smithi02683976-31 from 3964928 to 4751360 tid 2 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3495: writing smithi02683976-31 from 4751360 to 4767744 tid 3 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3480: done (9 left) 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3484: done (8 left) 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3488: done (7 left) 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3491: done (6 left) 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3492: done (5 left) 2024-09-13T14:30:45.190 INFO:tasks.rados.rados.0.smithi026.stdout:3496: snap_remove snap 298 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3496: done (5 left) 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3497: read oid 11 snap -1 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3497: expect (ObjNum 1048 snap 320 seq_num 1048) 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3498: copy_from oid 3 from oid 15 current snap is 334 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3490: finishing copy_from to smithi02683976-44 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1635 (ObjNum 1040 snap 319 seq_num 1040) dirty exists 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3493: finishing write tid 1 to smithi02683976-25 2024-09-13T14:30:46.067 INFO:tasks.rados.rados.0.smithi026.stdout:3489: expect (ObjNum 781 snap 242 seq_num 781) 2024-09-13T14:30:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:46 smithi026 ceph-mon[97022]: pgmap v705: 121 pgs: 121 active+clean; 605 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:30:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:46 smithi026 ceph-mon[97022]: osdmap e733: 8 total, 8 up, 8 in 2024-09-13T14:30:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:46 smithi026 ceph-mon[99461]: pgmap v705: 121 pgs: 121 active+clean; 605 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:30:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:46 smithi026 ceph-mon[99461]: osdmap e733: 8 total, 8 up, 8 in 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3493: finishing write tid 2 to smithi02683976-25 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3493: finishing write tid 3 to smithi02683976-25 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3493: finishing write tid 4 to smithi02683976-25 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3493: finishing write tid 5 to smithi02683976-25 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2011 (ObjNum 1094 snap 334 seq_num 1094) dirty exists 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3493: left oid 25 (ObjNum 1094 snap 334 seq_num 1094) 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3494: finishing write tid 1 to smithi02683976-23 2024-09-13T14:30:46.442 INFO:tasks.rados.rados.0.smithi026.stdout:3489: done (6 left) 2024-09-13T14:30:46.443 INFO:tasks.rados.rados.0.smithi026.stdout:3490: done (5 left) 2024-09-13T14:30:46.443 INFO:tasks.rados.rados.0.smithi026.stdout:3493: done (4 left) 2024-09-13T14:30:46.443 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 334 2024-09-13T14:30:46.443 INFO:tasks.rados.rados.0.smithi026.stdout:3494: finishing write tid 2 to smithi02683976-23 2024-09-13T14:30:46.443 INFO:tasks.rados.rados.0.smithi026.stdout:3499: seq_num 1097 ranges {1261568=778240,2039808=671744,2711552=131072} 2024-09-13T14:30:46.453 INFO:tasks.rados.rados.0.smithi026.stdout:3499: writing smithi02683976-30 from 1261568 to 2039808 tid 1 2024-09-13T14:30:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:46 smithi145 ceph-mon[89145]: pgmap v705: 121 pgs: 121 active+clean; 605 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:30:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:46 smithi145 ceph-mon[89145]: osdmap e733: 8 total, 8 up, 8 in 2024-09-13T14:30:46.456 INFO:tasks.rados.rados.0.smithi026.stdout:3499: writing smithi02683976-30 from 2039808 to 2711552 tid 2 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3499: writing smithi02683976-30 from 2711552 to 2842624 tid 3 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 334 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3494: finishing write tid 3 to smithi02683976-23 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3494: finishing write tid 4 to smithi02683976-23 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3494: finishing write tid 5 to smithi02683976-23 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1738 (ObjNum 1095 snap 334 seq_num 1095) dirty exists 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3494: left oid 23 (ObjNum 1095 snap 334 seq_num 1095) 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3495: finishing write tid 1 to smithi02683976-31 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3495: finishing write tid 2 to smithi02683976-31 2024-09-13T14:30:46.458 INFO:tasks.rados.rados.0.smithi026.stdout:3495: finishing write tid 3 to smithi02683976-31 2024-09-13T14:30:46.459 INFO:tasks.rados.rados.0.smithi026.stdout:3495: finishing write tid 4 to smithi02683976-31 2024-09-13T14:30:46.459 INFO:tasks.rados.rados.0.smithi026.stdout:3495: finishing write tid 5 to smithi02683976-31 2024-09-13T14:30:46.459 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1730 (ObjNum 1096 snap 334 seq_num 1096) dirty exists 2024-09-13T14:30:46.459 INFO:tasks.rados.rados.0.smithi026.stdout:3495: left oid 31 (ObjNum 1096 snap 334 seq_num 1096) 2024-09-13T14:30:46.459 INFO:tasks.rados.rados.0.smithi026.stdout:3500: seq_num 1098 ranges {2383872=794624,3178496=557056,3735552=245760} 2024-09-13T14:30:46.475 INFO:tasks.rados.rados.0.smithi026.stdout:3500: writing smithi02683976-36 from 2383872 to 3178496 tid 1 2024-09-13T14:30:46.478 INFO:tasks.rados.rados.0.smithi026.stdout:3500: writing smithi02683976-36 from 3178496 to 3735552 tid 2 2024-09-13T14:30:46.480 INFO:tasks.rados.rados.0.smithi026.stdout:3500: writing smithi02683976-36 from 3735552 to 3981312 tid 3 2024-09-13T14:30:46.480 INFO:tasks.rados.rados.0.smithi026.stdout:3494: done (5 left) 2024-09-13T14:30:46.480 INFO:tasks.rados.rados.0.smithi026.stdout:3495: done (4 left) 2024-09-13T14:30:46.480 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 334 2024-09-13T14:30:46.480 INFO:tasks.rados.rados.0.smithi026.stdout:3501: seq_num 1099 ranges {11747328=409600,12156928=442368,12599296=720896,13320192=40960} 2024-09-13T14:30:46.546 INFO:tasks.rados.rados.0.smithi026.stdout:3501: writing smithi02683976-22 from 11747328 to 12156928 tid 1 2024-09-13T14:30:46.548 INFO:tasks.rados.rados.0.smithi026.stdout:3501: writing smithi02683976-22 from 12156928 to 12599296 tid 2 2024-09-13T14:30:46.552 INFO:tasks.rados.rados.0.smithi026.stdout:3501: writing smithi02683976-22 from 12599296 to 13320192 tid 3 2024-09-13T14:30:46.553 INFO:tasks.rados.rados.0.smithi026.stdout:3501: writing smithi02683976-22 from 13320192 to 13361152 tid 4 2024-09-13T14:30:46.553 INFO:tasks.rados.rados.0.smithi026.stdout:3502: snap_remove snap 314 2024-09-13T14:30:47.072 INFO:tasks.rados.rados.0.smithi026.stdout:3502: done (5 left) 2024-09-13T14:30:47.072 INFO:tasks.rados.rados.0.smithi026.stdout:3503: snap_remove snap 313 2024-09-13T14:30:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:47 smithi026 ceph-mon[97022]: osdmap e734: 8 total, 8 up, 8 in 2024-09-13T14:30:47.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:47 smithi026 ceph-mon[99461]: osdmap e734: 8 total, 8 up, 8 in 2024-09-13T14:30:47.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:47 smithi145 ceph-mon[89145]: osdmap e734: 8 total, 8 up, 8 in 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:3503: done (5 left) 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:3504: rollback oid 34 current snap is 334 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 291 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:3505: rollback oid 38 current snap is 334 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 308 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:3506: read oid 25 snap -1 2024-09-13T14:30:48.077 INFO:tasks.rados.rados.0.smithi026.stdout:3506: expect (ObjNum 1094 snap 334 seq_num 1094) 2024-09-13T14:30:48.078 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 18 current snap is 334 2024-09-13T14:30:48.078 INFO:tasks.rados.rados.0.smithi026.stdout:3507: seq_num 1100 ranges {1040384=425984,1466368=729088,2195456=475136} 2024-09-13T14:30:48.086 INFO:tasks.rados.rados.0.smithi026.stdout:3507: writing smithi02683976-18 from 1040384 to 1466368 tid 1 2024-09-13T14:30:48.091 INFO:tasks.rados.rados.0.smithi026.stdout:3507: writing smithi02683976-18 from 1466368 to 2195456 tid 2 2024-09-13T14:30:48.096 INFO:tasks.rados.rados.0.smithi026.stdout:3507: writing smithi02683976-18 from 2195456 to 2670592 tid 3 2024-09-13T14:30:48.102 INFO:tasks.rados.rados.0.smithi026.stdout:3497: expect (ObjNum 1048 snap 320 seq_num 1048) 2024-09-13T14:30:48.128 INFO:tasks.rados.rados.0.smithi026.stdout:3498: finishing copy_from racing read to smithi02683976-3 2024-09-13T14:30:48.128 INFO:tasks.rados.rados.0.smithi026.stdout:3498: finishing copy_from to smithi02683976-3 2024-09-13T14:30:48.128 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 1828 (ObjNum 931 snap 290 seq_num 931) dirty exists 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3497: done (8 left) 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3498: done (7 left) 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3508: delete oid 28 current snap is 334 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: finishing write tid 1 to smithi02683976-30 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: finishing write tid 2 to smithi02683976-30 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: finishing write tid 3 to smithi02683976-30 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: finishing write tid 4 to smithi02683976-30 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: finishing write tid 5 to smithi02683976-30 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1832 (ObjNum 1097 snap 334 seq_num 1097) dirty exists 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3499: left oid 30 (ObjNum 1097 snap 334 seq_num 1097) 2024-09-13T14:30:48.129 INFO:tasks.rados.rados.0.smithi026.stdout:3500: finishing write tid 1 to smithi02683976-36 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3500: finishing write tid 2 to smithi02683976-36 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3500: finishing write tid 3 to smithi02683976-36 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3500: finishing write tid 4 to smithi02683976-36 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3500: finishing write tid 5 to smithi02683976-36 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2152 (ObjNum 1098 snap 334 seq_num 1098) dirty exists 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3500: left oid 36 (ObjNum 1098 snap 334 seq_num 1098) 2024-09-13T14:30:48.130 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 1 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 2 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 3 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 4 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 5 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: finishing write tid 6 to smithi02683976-22 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2010 (ObjNum 1099 snap 334 seq_num 1099) dirty exists 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3501: left oid 22 (ObjNum 1099 snap 334 seq_num 1099) 2024-09-13T14:30:48.131 INFO:tasks.rados.rados.0.smithi026.stdout:3504: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:30:48.132 INFO:tasks.rados.rados.0.smithi026.stdout:3504: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:30:48.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1930 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:30:48.132 INFO:tasks.rados.rados.0.smithi026.stdout:3505: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:30:48.132 INFO:tasks.rados.rados.0.smithi026.stdout:3505: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:30:48.132 INFO:tasks.rados.rados.0.smithi026.stdout:3505: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 1759 (ObjNum 977 snap 306 seq_num 977) dirty exists 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: finishing write tid 1 to smithi02683976-18 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: finishing write tid 2 to smithi02683976-18 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: finishing write tid 3 to smithi02683976-18 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: finishing write tid 4 to smithi02683976-18 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: finishing write tid 5 to smithi02683976-18 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 1764 (ObjNum 1100 snap 334 seq_num 1100) dirty exists 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3507: left oid 18 (ObjNum 1100 snap 334 seq_num 1100) 2024-09-13T14:30:48.133 INFO:tasks.rados.rados.0.smithi026.stdout:3506: expect (ObjNum 1094 snap 334 seq_num 1094) 2024-09-13T14:30:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:48 smithi026 ceph-mon[97022]: pgmap v708: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 610 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-13T14:30:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:48 smithi026 ceph-mon[97022]: osdmap e735: 8 total, 8 up, 8 in 2024-09-13T14:30:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:48 smithi026 ceph-mon[99461]: pgmap v708: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 610 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-13T14:30:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:48 smithi026 ceph-mon[99461]: osdmap e735: 8 total, 8 up, 8 in 2024-09-13T14:30:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:48 smithi145 ceph-mon[89145]: pgmap v708: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 610 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-13T14:30:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:48 smithi145 ceph-mon[89145]: osdmap e735: 8 total, 8 up, 8 in 2024-09-13T14:30:48.502 INFO:tasks.rados.rados.0.smithi026.stdout:3499: done (7 left) 2024-09-13T14:30:48.502 INFO:tasks.rados.rados.0.smithi026.stdout:3500: done (6 left) 2024-09-13T14:30:48.502 INFO:tasks.rados.rados.0.smithi026.stdout:3501: done (5 left) 2024-09-13T14:30:48.502 INFO:tasks.rados.rados.0.smithi026.stdout:3504: done (4 left) 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3505: done (3 left) 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3506: done (2 left) 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3507: done (1 left) 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3508: done (0 left) 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3509: rollback oid 20 current snap is 334 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 301 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3510: rollback oid 26 current snap is 334 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 26 to 303 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3511: read oid 1 snap 308 2024-09-13T14:30:48.503 INFO:tasks.rados.rados.0.smithi026.stdout:3511: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:48.504 INFO:tasks.rados.rados.0.smithi026.stdout:3512: setattr oid 16 current snap is 334 2024-09-13T14:30:48.504 INFO:tasks.rados.rados.0.smithi026.stdout:3513: read oid 29 snap -1 2024-09-13T14:30:48.504 INFO:tasks.rados.rados.0.smithi026.stdout:3513: expect (ObjNum 1093 snap 334 seq_num 1093) 2024-09-13T14:30:48.505 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 334 2024-09-13T14:30:48.505 INFO:tasks.rados.rados.0.smithi026.stdout:3514: seq_num 1102 ranges {1400832=458752,1859584=491520,2351104=491520,2842624=221184} 2024-09-13T14:30:48.513 INFO:tasks.rados.rados.0.smithi026.stdout:3514: writing smithi02683976-8 from 1400832 to 1859584 tid 1 2024-09-13T14:30:48.516 INFO:tasks.rados.rados.0.smithi026.stdout:3514: writing smithi02683976-8 from 1859584 to 2351104 tid 2 2024-09-13T14:30:48.519 INFO:tasks.rados.rados.0.smithi026.stdout:3514: writing smithi02683976-8 from 2351104 to 2842624 tid 3 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:3514: writing smithi02683976-8 from 2842624 to 3063808 tid 4 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 334 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:3510: finishing rollback tid 0 to smithi02683976-26 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:3510: finishing rollback tid 1 to smithi02683976-26 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:3510: finishing rollback tid 2 to smithi02683976-26 2024-09-13T14:30:48.521 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 2013 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:48.522 INFO:tasks.rados.rados.0.smithi026.stdout:3509: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:30:48.522 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 0 (ObjNum 161 snap 0 seq_num 430) dirty dne 2024-09-13T14:30:48.522 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 2015 (ObjNum 1069 snap 331 seq_num 1069) dirty exists 2024-09-13T14:30:48.522 INFO:tasks.rados.rados.0.smithi026.stdout:3513: expect (ObjNum 1093 snap 334 seq_num 1093) 2024-09-13T14:30:48.561 INFO:tasks.rados.rados.0.smithi026.stdout:3511: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:30:48.727 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 1 to smithi02683976-8 2024-09-13T14:30:48.727 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 2 to smithi02683976-8 2024-09-13T14:30:48.727 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 3 to smithi02683976-8 2024-09-13T14:30:48.727 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 4 to smithi02683976-8 2024-09-13T14:30:48.727 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 5 to smithi02683976-8 2024-09-13T14:30:48.728 INFO:tasks.rados.rados.0.smithi026.stdout:3514: finishing write tid 6 to smithi02683976-8 2024-09-13T14:30:48.728 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1524 (ObjNum 1102 snap 334 seq_num 1102) dirty exists 2024-09-13T14:30:48.728 INFO:tasks.rados.rados.0.smithi026.stdout:3514: left oid 8 (ObjNum 1102 snap 334 seq_num 1102) 2024-09-13T14:30:48.728 INFO:tasks.rados.rados.0.smithi026.stdout:3515: seq_num 1103 ranges {4767744=475136,5242880=425984,5668864=696320,6365184=81920} 2024-09-13T14:30:48.755 INFO:tasks.rados.rados.0.smithi026.stdout:3515: writing smithi02683976-31 from 4767744 to 5242880 tid 1 2024-09-13T14:30:48.757 INFO:tasks.rados.rados.0.smithi026.stdout:3515: writing smithi02683976-31 from 5242880 to 5668864 tid 2 2024-09-13T14:30:48.761 INFO:tasks.rados.rados.0.smithi026.stdout:3515: writing smithi02683976-31 from 5668864 to 6365184 tid 3 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3515: writing smithi02683976-31 from 6365184 to 6447104 tid 4 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3509: done (6 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3510: done (5 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3511: done (4 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3512: done (3 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3513: done (2 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3514: done (1 left) 2024-09-13T14:30:48.762 INFO:tasks.rados.rados.0.smithi026.stdout:3516: read oid 24 snap -1 2024-09-13T14:30:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:3516: expect (ObjNum 1061 snap 326 seq_num 1061) 2024-09-13T14:30:48.763 INFO:tasks.rados.rados.0.smithi026.stdout:3517: snap_remove snap 303 2024-09-13T14:30:49.019 INFO:tasks.rados.rados.0.smithi026.stdout:3517: done (2 left) 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3518: rmattr oid 41 current snap is 334 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 1 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 2 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 3 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 4 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 5 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: finishing write tid 6 to smithi02683976-31 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1751 (ObjNum 1103 snap 334 seq_num 1103) dirty exists 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3515: left oid 31 (ObjNum 1103 snap 334 seq_num 1103) 2024-09-13T14:30:49.020 INFO:tasks.rados.rados.0.smithi026.stdout:3516: expect (ObjNum 1061 snap 326 seq_num 1061) 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1931 (ObjNum 949 snap 296 seq_num 949) dirty exists 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:3515: done (2 left) 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:3516: done (1 left) 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:3518: done (0 left) 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 21 current snap is 334 2024-09-13T14:30:49.308 INFO:tasks.rados.rados.0.smithi026.stdout:3519: seq_num 1104 ranges {0=499712,499712=712704,1212416=491520} 2024-09-13T14:30:49.310 INFO:tasks.rados.rados.0.smithi026.stdout:3519: writing smithi02683976-21 from 0 to 499712 tid 1 2024-09-13T14:30:49.315 INFO:tasks.rados.rados.0.smithi026.stdout:3519: writing smithi02683976-21 from 499712 to 1212416 tid 2 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3519: writing smithi02683976-21 from 1212416 to 1703936 tid 3 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3520: read oid 43 snap -1 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3520: expect deleted 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3521: read oid 47 snap -1 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3521: expect (ObjNum 1063 snap 326 seq_num 1063) 2024-09-13T14:30:49.321 INFO:tasks.rados.rados.0.smithi026.stdout:3522: rmattr oid 5 current snap is 334 2024-09-13T14:30:49.322 INFO:tasks.rados.rados.0.smithi026.stdout:3519: finishing write tid 1 to smithi02683976-21 2024-09-13T14:30:49.322 INFO:tasks.rados.rados.0.smithi026.stdout:3523: rollback oid 4 current snap is 334 2024-09-13T14:30:49.322 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 4 to 285 2024-09-13T14:30:49.322 INFO:tasks.rados.rados.0.smithi026.stdout:3524: delete oid 49 current snap is 334 2024-09-13T14:30:49.322 INFO:tasks.rados.rados.0.smithi026.stdout:3520: done (5 left) 2024-09-13T14:30:49.323 INFO:tasks.rados.rados.0.smithi026.stdout:3524: done (4 left) 2024-09-13T14:30:49.323 INFO:tasks.rados.rados.0.smithi026.stdout:3525: delete oid 27 current snap is 334 2024-09-13T14:30:49.323 INFO:tasks.rados.rados.0.smithi026.stdout:3519: finishing write tid 2 to smithi02683976-21 2024-09-13T14:30:49.323 INFO:tasks.rados.rados.0.smithi026.stdout:3521: expect (ObjNum 1063 snap 326 seq_num 1063) 2024-09-13T14:30:49.351 INFO:tasks.rados.rados.0.smithi026.stdout:3519: finishing write tid 3 to smithi02683976-21 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3523: finishing rollback tid 0 to smithi02683976-4 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3519: finishing write tid 4 to smithi02683976-21 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3523: finishing rollback tid 1 to smithi02683976-4 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3523: finishing rollback tid 2 to smithi02683976-4 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1835 (ObjNum 871 snap 269 seq_num 871) dirty exists 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3521: done (4 left) 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3523: done (3 left) 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3525: done (2 left) 2024-09-13T14:30:49.352 INFO:tasks.rados.rados.0.smithi026.stdout:3526: snap_remove snap 331 2024-09-13T14:30:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:49 smithi026 ceph-mon[97022]: osdmap e736: 8 total, 8 up, 8 in 2024-09-13T14:30:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:49 smithi026 ceph-mon[97022]: osdmap e737: 8 total, 8 up, 8 in 2024-09-13T14:30:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:49 smithi026 ceph-mon[99461]: osdmap e736: 8 total, 8 up, 8 in 2024-09-13T14:30:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:49 smithi026 ceph-mon[99461]: osdmap e737: 8 total, 8 up, 8 in 2024-09-13T14:30:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:49 smithi145 ceph-mon[89145]: osdmap e736: 8 total, 8 up, 8 in 2024-09-13T14:30:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:49 smithi145 ceph-mon[89145]: osdmap e737: 8 total, 8 up, 8 in 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:3526: done (2 left) 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:3527: delete oid 19 current snap is 334 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2184 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:3519: finishing write tid 5 to smithi02683976-21 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1766 (ObjNum 1104 snap 334 seq_num 1104) dirty exists 2024-09-13T14:30:50.084 INFO:tasks.rados.rados.0.smithi026.stdout:3519: left oid 21 (ObjNum 1104 snap 334 seq_num 1104) 2024-09-13T14:30:50.097 INFO:tasks.rados.rados.0.smithi026.stdout:3519: done (2 left) 2024-09-13T14:30:50.097 INFO:tasks.rados.rados.0.smithi026.stdout:3522: done (1 left) 2024-09-13T14:30:50.097 INFO:tasks.rados.rados.0.smithi026.stdout:3527: done (0 left) 2024-09-13T14:30:50.098 INFO:tasks.rados.rados.0.smithi026.stdout:3528: snap_create 2024-09-13T14:30:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:50 smithi026 ceph-mon[97022]: pgmap v711: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 611 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:50 smithi026 ceph-mon[99461]: pgmap v711: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 611 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:50 smithi145 ceph-mon[89145]: pgmap v711: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 611 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-13T14:30:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:30:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:30:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:51.096 INFO:tasks.rados.rados.0.smithi026.stdout:3528: done (0 left) 2024-09-13T14:30:51.096 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 8 current snap is 335 2024-09-13T14:30:51.096 INFO:tasks.rados.rados.0.smithi026.stdout:3529: seq_num 1108 ranges {3063808=565248,3629056=647168,4276224=483328,4759552=73728} 2024-09-13T14:30:51.116 INFO:tasks.rados.rados.0.smithi026.stdout:3529: writing smithi02683976-8 from 3063808 to 3629056 tid 1 2024-09-13T14:30:51.120 INFO:tasks.rados.rados.0.smithi026.stdout:3529: writing smithi02683976-8 from 3629056 to 4276224 tid 2 2024-09-13T14:30:51.122 INFO:tasks.rados.rados.0.smithi026.stdout:3529: writing smithi02683976-8 from 4276224 to 4759552 tid 3 2024-09-13T14:30:51.123 INFO:tasks.rados.rados.0.smithi026.stdout:3529: writing smithi02683976-8 from 4759552 to 4833280 tid 4 2024-09-13T14:30:51.123 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 335 2024-09-13T14:30:51.123 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 1 to smithi02683976-8 2024-09-13T14:30:51.123 INFO:tasks.rados.rados.0.smithi026.stdout:3530: seq_num 1109 ranges {1515520=581632,2097152=532480,2629632=647168,3276800=24576} 2024-09-13T14:30:51.135 INFO:tasks.rados.rados.0.smithi026.stdout:3530: writing smithi02683976-29 from 1515520 to 2097152 tid 1 2024-09-13T14:30:51.138 INFO:tasks.rados.rados.0.smithi026.stdout:3530: writing smithi02683976-29 from 2097152 to 2629632 tid 2 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3530: writing smithi02683976-29 from 2629632 to 3276800 tid 3 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3530: writing smithi02683976-29 from 3276800 to 3301376 tid 4 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3531: copy_from oid 12 from oid 17 current snap is 335 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 2 to smithi02683976-8 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 3 to smithi02683976-8 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 4 to smithi02683976-8 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 5 to smithi02683976-8 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:3529: finishing write tid 6 to smithi02683976-8 2024-09-13T14:30:51.143 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1529 (ObjNum 1108 snap 335 seq_num 1108) dirty exists 2024-09-13T14:30:51.144 INFO:tasks.rados.rados.0.smithi026.stdout:3529: left oid 8 (ObjNum 1108 snap 335 seq_num 1108) 2024-09-13T14:30:51.144 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 1 to smithi02683976-29 2024-09-13T14:30:51.144 INFO:tasks.rados.rados.0.smithi026.stdout:3529: done (2 left) 2024-09-13T14:30:51.144 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 335 2024-09-13T14:30:51.144 INFO:tasks.rados.rados.0.smithi026.stdout:3532: seq_num 1110 ranges {13361152=598016,13959168=466944,14426112=450560,14876672=286720} 2024-09-13T14:30:51.219 INFO:tasks.rados.rados.0.smithi026.stdout:3532: writing smithi02683976-22 from 13361152 to 13959168 tid 1 2024-09-13T14:30:51.221 INFO:tasks.rados.rados.0.smithi026.stdout:3532: writing smithi02683976-22 from 13959168 to 14426112 tid 2 2024-09-13T14:30:51.223 INFO:tasks.rados.rados.0.smithi026.stdout:3532: writing smithi02683976-22 from 14426112 to 14876672 tid 3 2024-09-13T14:30:51.225 INFO:tasks.rados.rados.0.smithi026.stdout:3532: writing smithi02683976-22 from 14876672 to 15163392 tid 4 2024-09-13T14:30:51.225 INFO:tasks.rados.rados.0.smithi026.stdout:3533: snap_remove snap 326 2024-09-13T14:30:51.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:51 smithi026 ceph-mon[97022]: osdmap e738: 8 total, 8 up, 8 in 2024-09-13T14:30:51.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:51 smithi026 ceph-mon[99461]: osdmap e738: 8 total, 8 up, 8 in 2024-09-13T14:30:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:51 smithi145 ceph-mon[89145]: osdmap e738: 8 total, 8 up, 8 in 2024-09-13T14:30:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:30:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:30:51] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3533: done (3 left) 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3534: rollback oid 40 current snap is 335 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 40 to 256 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 2 to smithi02683976-29 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 3 to smithi02683976-29 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 4 to smithi02683976-29 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 5 to smithi02683976-29 2024-09-13T14:30:52.104 INFO:tasks.rados.rados.0.smithi026.stdout:3530: finishing write tid 6 to smithi02683976-29 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 2021 (ObjNum 1109 snap 335 seq_num 1109) dirty exists 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3530: left oid 29 (ObjNum 1109 snap 335 seq_num 1109) 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3531: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3531: finishing copy_from to smithi02683976-12 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 1229 (ObjNum 1064 snap 327 seq_num 1064) dirty exists 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 1 to smithi02683976-22 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 2 to smithi02683976-22 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 3 to smithi02683976-22 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 4 to smithi02683976-22 2024-09-13T14:30:52.105 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 5 to smithi02683976-22 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3532: finishing write tid 6 to smithi02683976-22 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2018 (ObjNum 1110 snap 335 seq_num 1110) dirty exists 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3532: left oid 22 (ObjNum 1110 snap 335 seq_num 1110) 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3530: done (3 left) 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3531: done (2 left) 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3532: done (1 left) 2024-09-13T14:30:52.106 INFO:tasks.rados.rados.0.smithi026.stdout:3535: snap_remove snap 324 2024-09-13T14:30:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:52 smithi026 ceph-mon[97022]: pgmap v714: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 657 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 6.3 MiB/s wr, 33 op/s 2024-09-13T14:30:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:52 smithi026 ceph-mon[97022]: osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:30:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:52 smithi026 ceph-mon[99461]: pgmap v714: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 657 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 6.3 MiB/s wr, 33 op/s 2024-09-13T14:30:52.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:52 smithi026 ceph-mon[99461]: osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:30:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:52 smithi145 ceph-mon[89145]: pgmap v714: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 657 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 6.3 MiB/s wr, 33 op/s 2024-09-13T14:30:52.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:52 smithi145 ceph-mon[89145]: osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:30:53.108 INFO:tasks.rados.rados.0.smithi026.stdout:3535: done (1 left) 2024-09-13T14:30:53.108 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 9 current snap is 335 2024-09-13T14:30:53.108 INFO:tasks.rados.rados.0.smithi026.stdout:3536: seq_num 1111 ranges {1212416=614400,1826816=753664,2580480=450560} 2024-09-13T14:30:53.119 INFO:tasks.rados.rados.0.smithi026.stdout:3536: writing smithi02683976-9 from 1212416 to 1826816 tid 1 2024-09-13T14:30:53.125 INFO:tasks.rados.rados.0.smithi026.stdout:3536: writing smithi02683976-9 from 1826816 to 2580480 tid 2 2024-09-13T14:30:53.129 INFO:tasks.rados.rados.0.smithi026.stdout:3536: writing smithi02683976-9 from 2580480 to 3031040 tid 3 2024-09-13T14:30:53.129 INFO:tasks.rados.rados.0.smithi026.stdout:3537: snap_create 2024-09-13T14:30:53.129 INFO:tasks.rados.rados.0.smithi026.stdout:3534: finishing rollback tid 1 to smithi02683976-40 2024-09-13T14:30:53.129 INFO:tasks.rados.rados.0.smithi026.stdout:3534: finishing rollback tid 2 to smithi02683976-40 2024-09-13T14:30:53.129 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 1651 (ObjNum 813 snap 254 seq_num 813) dirty exists 2024-09-13T14:30:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:3536: finishing write tid 1 to smithi02683976-9 2024-09-13T14:30:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:3536: finishing write tid 2 to smithi02683976-9 2024-09-13T14:30:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:3536: finishing write tid 3 to smithi02683976-9 2024-09-13T14:30:53.134 INFO:tasks.rados.rados.0.smithi026.stdout:3536: finishing write tid 4 to smithi02683976-9 2024-09-13T14:30:53.135 INFO:tasks.rados.rados.0.smithi026.stdout:3536: finishing write tid 5 to smithi02683976-9 2024-09-13T14:30:53.136 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1789 (ObjNum 1111 snap 335 seq_num 1111) dirty exists 2024-09-13T14:30:53.136 INFO:tasks.rados.rados.0.smithi026.stdout:3536: left oid 9 (ObjNum 1111 snap 335 seq_num 1111) 2024-09-13T14:30:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:53 smithi026 ceph-mon[97022]: osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:30:53.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:53 smithi026 ceph-mon[99461]: osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:30:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:53 smithi145 ceph-mon[89145]: osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:30:54.023 INFO:tasks.rados.rados.0.smithi026.stdout:3534: done (2 left) 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3536: done (1 left) 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3537: done (0 left) 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3538: read oid 31 snap -1 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3538: expect (ObjNum 1103 snap 334 seq_num 1103) 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3539: read oid 45 snap -1 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3539: expect (ObjNum 1054 snap 323 seq_num 1054) 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3540: read oid 38 snap 285 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3540: expect deleted 2024-09-13T14:30:54.024 INFO:tasks.rados.rados.0.smithi026.stdout:3541: delete oid 32 current snap is 336 2024-09-13T14:30:54.033 INFO:tasks.rados.rados.0.smithi026.stdout:3540: done (3 left) 2024-09-13T14:30:54.033 INFO:tasks.rados.rados.0.smithi026.stdout:3541: done (2 left) 2024-09-13T14:30:54.033 INFO:tasks.rados.rados.0.smithi026.stdout:3542: copy_from oid 40 from oid 26 current snap is 336 2024-09-13T14:30:54.033 INFO:tasks.rados.rados.0.smithi026.stdout:3543: snap_remove snap 308 2024-09-13T14:30:54.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[97022]: pgmap v717: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 660 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.7 MiB/s wr, 44 op/s 2024-09-13T14:30:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[97022]: osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:30:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[97022]: osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:30:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[99461]: pgmap v717: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 660 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.7 MiB/s wr, 44 op/s 2024-09-13T14:30:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[99461]: osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:30:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:54 smithi026 ceph-mon[99461]: osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:30:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:54 smithi145 ceph-mon[89145]: pgmap v717: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 660 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 6.7 MiB/s wr, 44 op/s 2024-09-13T14:30:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:54 smithi145 ceph-mon[89145]: osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:30:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:54 smithi145 ceph-mon[89145]: osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:30:55.036 INFO:tasks.rados.rados.0.smithi026.stdout:3539: expect (ObjNum 1054 snap 323 seq_num 1054) 2024-09-13T14:30:55.106 INFO:tasks.rados.rados.0.smithi026.stdout:3539: done (3 left) 2024-09-13T14:30:55.107 INFO:tasks.rados.rados.0.smithi026.stdout:3543: done (2 left) 2024-09-13T14:30:55.107 INFO:tasks.rados.rados.0.smithi026.stdout:3544: snap_remove snap 329 2024-09-13T14:30:56.111 INFO:tasks.rados.rados.0.smithi026.stdout:3544: done (2 left) 2024-09-13T14:30:56.111 INFO:tasks.rados.rados.0.smithi026.stdout:3545: rollback oid 44 current snap is 336 2024-09-13T14:30:56.112 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 44 to 316 2024-09-13T14:30:56.112 INFO:tasks.rados.rados.0.smithi026.stdout:3538: expect (ObjNum 1103 snap 334 seq_num 1103) 2024-09-13T14:30:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:56 smithi026 ceph-mon[99461]: pgmap v720: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 663 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-13T14:30:56.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:56 smithi026 ceph-mon[99461]: osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:30:56.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:56 smithi026 ceph-mon[97022]: pgmap v720: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 663 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-13T14:30:56.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:56 smithi026 ceph-mon[97022]: osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3542: finishing copy_from racing read to smithi02683976-40 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3542: finishing copy_from to smithi02683976-40 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 2013 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3538: done (2 left) 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3542: done (1 left) 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3546: copy_from oid 23 from oid 29 current snap is 336 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3545: finishing rollback tid 0 to smithi02683976-44 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3545: finishing rollback tid 1 to smithi02683976-44 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3545: finishing rollback tid 2 to smithi02683976-44 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1649 (ObjNum 963 snap 302 seq_num 963) dirty exists 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3545: done (1 left) 2024-09-13T14:30:56.450 INFO:tasks.rados.rados.0.smithi026.stdout:3547: read oid 14 snap 301 2024-09-13T14:30:56.451 INFO:tasks.rados.rados.0.smithi026.stdout:3547: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-13T14:30:56.451 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 336 2024-09-13T14:30:56.451 INFO:tasks.rados.rados.0.smithi026.stdout:3548: seq_num 1113 ranges {0=647168,647168=524288,1171456=622592,1794048=57344} 2024-09-13T14:30:56.452 INFO:tasks.rados.rados.0.smithi026.stdout:3548: writing smithi02683976-19 from 0 to 647168 tid 1 2024-09-13T14:30:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:56 smithi145 ceph-mon[89145]: pgmap v720: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 663 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-13T14:30:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:56 smithi145 ceph-mon[89145]: osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:30:56.456 INFO:tasks.rados.rados.0.smithi026.stdout:3548: writing smithi02683976-19 from 647168 to 1171456 tid 2 2024-09-13T14:30:56.460 INFO:tasks.rados.rados.0.smithi026.stdout:3548: writing smithi02683976-19 from 1171456 to 1794048 tid 3 2024-09-13T14:30:56.461 INFO:tasks.rados.rados.0.smithi026.stdout:3548: writing smithi02683976-19 from 1794048 to 1851392 tid 4 2024-09-13T14:30:56.461 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 1 to smithi02683976-19 2024-09-13T14:30:56.461 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 2 to smithi02683976-19 2024-09-13T14:30:56.461 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 336 2024-09-13T14:30:56.461 INFO:tasks.rados.rados.0.smithi026.stdout:3549: seq_num 1114 ranges {12083200=663552,12746752=802816,13549568=401408} 2024-09-13T14:30:56.531 INFO:tasks.rados.rados.0.smithi026.stdout:3549: writing smithi02683976-40 from 12083200 to 12746752 tid 1 2024-09-13T14:30:56.536 INFO:tasks.rados.rados.0.smithi026.stdout:3549: writing smithi02683976-40 from 12746752 to 13549568 tid 2 2024-09-13T14:30:56.538 INFO:tasks.rados.rados.0.smithi026.stdout:3549: writing smithi02683976-40 from 13549568 to 13950976 tid 3 2024-09-13T14:30:56.538 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 336 2024-09-13T14:30:56.538 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 3 to smithi02683976-19 2024-09-13T14:30:56.538 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 4 to smithi02683976-19 2024-09-13T14:30:56.539 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 5 to smithi02683976-19 2024-09-13T14:30:56.539 INFO:tasks.rados.rados.0.smithi026.stdout:3550: seq_num 1115 ranges {0=679936,679936=688128,1368064=516096} 2024-09-13T14:30:56.543 INFO:tasks.rados.rados.0.smithi026.stdout:3550: writing smithi02683976-49 from 0 to 679936 tid 1 2024-09-13T14:30:56.546 INFO:tasks.rados.rados.0.smithi026.stdout:3550: writing smithi02683976-49 from 679936 to 1368064 tid 2 2024-09-13T14:30:56.550 INFO:tasks.rados.rados.0.smithi026.stdout:3550: writing smithi02683976-49 from 1368064 to 1884160 tid 3 2024-09-13T14:30:56.550 INFO:tasks.rados.rados.0.smithi026.stdout:3551: delete oid 12 current snap is 336 2024-09-13T14:30:56.550 INFO:tasks.rados.rados.0.smithi026.stdout:3546: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:30:56.552 INFO:tasks.rados.rados.0.smithi026.stdout:3548: finishing write tid 6 to smithi02683976-19 2024-09-13T14:30:56.552 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2026 (ObjNum 1113 snap 336 seq_num 1113) dirty exists 2024-09-13T14:30:56.552 INFO:tasks.rados.rados.0.smithi026.stdout:3548: left oid 19 (ObjNum 1113 snap 336 seq_num 1113) 2024-09-13T14:30:56.552 INFO:tasks.rados.rados.0.smithi026.stdout:3547: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-13T14:30:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:57 smithi026 ceph-mon[97022]: osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:30:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:57 smithi026 ceph-mon[99461]: osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:30:57.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:57 smithi145 ceph-mon[89145]: osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:30:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:58.399 INFO:tasks.rados.rados.0.smithi026.stdout:3546: finishing copy_from to smithi02683976-23 2024-09-13T14:30:58.399 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 2021 (ObjNum 1109 snap 335 seq_num 1109) dirty exists 2024-09-13T14:30:58.399 INFO:tasks.rados.rados.0.smithi026.stdout:3549: finishing write tid 1 to smithi02683976-40 2024-09-13T14:30:58.399 INFO:tasks.rados.rados.0.smithi026.stdout:3549: finishing write tid 2 to smithi02683976-40 2024-09-13T14:30:58.399 INFO:tasks.rados.rados.0.smithi026.stdout:3546: done (5 left) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3547: done (4 left) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3548: done (3 left) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3551: done (2 left) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3552: read oid 25 snap 334 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3552: expect (ObjNum 1094 snap 334 seq_num 1094) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3553: delete oid 47 current snap is 336 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3549: finishing write tid 3 to smithi02683976-40 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3549: finishing write tid 4 to smithi02683976-40 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3549: finishing write tid 5 to smithi02683976-40 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 2017 (ObjNum 1114 snap 336 seq_num 1114) dirty exists 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3549: left oid 40 (ObjNum 1114 snap 336 seq_num 1114) 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3550: finishing write tid 1 to smithi02683976-49 2024-09-13T14:30:58.400 INFO:tasks.rados.rados.0.smithi026.stdout:3550: finishing write tid 2 to smithi02683976-49 2024-09-13T14:30:58.401 INFO:tasks.rados.rados.0.smithi026.stdout:3550: finishing write tid 3 to smithi02683976-49 2024-09-13T14:30:58.401 INFO:tasks.rados.rados.0.smithi026.stdout:3550: finishing write tid 4 to smithi02683976-49 2024-09-13T14:30:58.401 INFO:tasks.rados.rados.0.smithi026.stdout:3550: finishing write tid 5 to smithi02683976-49 2024-09-13T14:30:58.401 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2021 (ObjNum 1115 snap 336 seq_num 1115) dirty exists 2024-09-13T14:30:58.401 INFO:tasks.rados.rados.0.smithi026.stdout:3550: left oid 49 (ObjNum 1115 snap 336 seq_num 1115) 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:3549: done (3 left) 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:3550: done (2 left) 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:3553: done (1 left) 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:3554: rollback oid 31 current snap is 336 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 31 to 327 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 336 2024-09-13T14:30:58.402 INFO:tasks.rados.rados.0.smithi026.stdout:3555: seq_num 1118 ranges {1851392=729088,2580480=794624,3375104=409600} 2024-09-13T14:30:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:30:58 smithi026 ceph-mon[99461]: pgmap v723: 121 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 114 active+clean; 681 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 5.6 MiB/s wr, 8 op/s 2024-09-13T14:30:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:30:58 smithi026 ceph-mon[97022]: pgmap v723: 121 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 114 active+clean; 681 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 5.6 MiB/s wr, 8 op/s 2024-09-13T14:30:58.416 INFO:tasks.rados.rados.0.smithi026.stdout:3555: writing smithi02683976-19 from 1851392 to 2580480 tid 1 2024-09-13T14:30:58.421 INFO:tasks.rados.rados.0.smithi026.stdout:3555: writing smithi02683976-19 from 2580480 to 3375104 tid 2 2024-09-13T14:30:58.424 INFO:tasks.rados.rados.0.smithi026.stdout:3555: writing smithi02683976-19 from 3375104 to 3784704 tid 3 2024-09-13T14:30:58.424 INFO:tasks.rados.rados.0.smithi026.stdout:3556: snap_create 2024-09-13T14:30:58.424 INFO:tasks.rados.rados.0.smithi026.stdout:3554: finishing rollback tid 0 to smithi02683976-31 2024-09-13T14:30:58.424 INFO:tasks.rados.rados.0.smithi026.stdout:3554: finishing rollback tid 1 to smithi02683976-31 2024-09-13T14:30:58.425 INFO:tasks.rados.rados.0.smithi026.stdout:3554: finishing rollback tid 2 to smithi02683976-31 2024-09-13T14:30:58.425 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1809 (ObjNum 963 snap 302 seq_num 963) dirty exists 2024-09-13T14:30:58.425 INFO:tasks.rados.rados.0.smithi026.stdout:3552: expect (ObjNum 1094 snap 334 seq_num 1094) 2024-09-13T14:30:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:30:58 smithi145 ceph-mon[89145]: pgmap v723: 121 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 114 active+clean; 681 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 5.6 MiB/s wr, 8 op/s 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: finishing write tid 1 to smithi02683976-19 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: finishing write tid 2 to smithi02683976-19 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: finishing write tid 3 to smithi02683976-19 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: finishing write tid 4 to smithi02683976-19 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: finishing write tid 5 to smithi02683976-19 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2030 (ObjNum 1118 snap 336 seq_num 1118) dirty exists 2024-09-13T14:30:58.798 INFO:tasks.rados.rados.0.smithi026.stdout:3555: left oid 19 (ObjNum 1118 snap 336 seq_num 1118) 2024-09-13T14:30:59.023 INFO:tasks.rados.rados.0.smithi026.stdout:3552: done (3 left) 2024-09-13T14:30:59.023 INFO:tasks.rados.rados.0.smithi026.stdout:3554: done (2 left) 2024-09-13T14:30:59.023 INFO:tasks.rados.rados.0.smithi026.stdout:3555: done (1 left) 2024-09-13T14:30:59.024 INFO:tasks.rados.rados.0.smithi026.stdout:3556: done (0 left) 2024-09-13T14:30:59.024 INFO:tasks.rados.rados.0.smithi026.stdout:3557: rollback oid 16 current snap is 337 2024-09-13T14:30:59.024 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 16 to 256 2024-09-13T14:30:59.024 INFO:tasks.rados.rados.0.smithi026.stdout:3558: setattr oid 43 current snap is 337 2024-09-13T14:30:59.025 INFO:tasks.rados.rados.0.smithi026.stdout:3559: delete oid 40 current snap is 337 2024-09-13T14:30:59.029 INFO:tasks.rados.rados.0.smithi026.stdout:3559: done (2 left) 2024-09-13T14:30:59.030 INFO:tasks.rados.rados.0.smithi026.stdout:3560: copy_from oid 13 from oid 49 current snap is 337 2024-09-13T14:30:59.030 INFO:tasks.rados.rados.0.smithi026.stdout:3561: setattr oid 22 current snap is 337 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:3557: finishing rollback tid 0 to smithi02683976-16 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:3557: finishing rollback tid 1 to smithi02683976-16 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:3557: finishing rollback tid 2 to smithi02683976-16 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 16 v 2033 (ObjNum 798 snap 245 seq_num 798) dirty exists 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:3557: done (3 left) 2024-09-13T14:30:59.032 INFO:tasks.rados.rados.0.smithi026.stdout:3562: copy_from oid 9 from oid 6 current snap is 337 2024-09-13T14:30:59.033 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 337 2024-09-13T14:30:59.033 INFO:tasks.rados.rados.0.smithi026.stdout:3563: seq_num 1120 ranges {2842624=761856,3604480=557056,4161536=425984,4587520=221184} 2024-09-13T14:30:59.052 INFO:tasks.rados.rados.0.smithi026.stdout:3563: writing smithi02683976-30 from 2842624 to 3604480 tid 1 2024-09-13T14:30:59.055 INFO:tasks.rados.rados.0.smithi026.stdout:3563: writing smithi02683976-30 from 3604480 to 4161536 tid 2 2024-09-13T14:30:59.057 INFO:tasks.rados.rados.0.smithi026.stdout:3563: writing smithi02683976-30 from 4161536 to 4587520 tid 3 2024-09-13T14:30:59.059 INFO:tasks.rados.rados.0.smithi026.stdout:3563: writing smithi02683976-30 from 4587520 to 4808704 tid 4 2024-09-13T14:30:59.059 INFO:tasks.rados.rados.0.smithi026.stdout:3564: snap_create 2024-09-13T14:30:59.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2034 (ObjNum 65 snap 0 seq_num 1684563041) dirty exists 2024-09-13T14:30:59.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2020 (ObjNum 1110 snap 335 seq_num 1110) dirty exists 2024-09-13T14:30:59.059 INFO:tasks.rados.rados.0.smithi026.stdout:3562: finishing copy_from racing read to smithi02683976-9 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:3560: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:3562: finishing copy_from to smithi02683976-9 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 1648 (ObjNum 1039 snap 319 seq_num 1039) dirty exists 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:3560: finishing copy_from to smithi02683976-13 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 2021 (ObjNum 1115 snap 336 seq_num 1115) dirty exists 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 1 to smithi02683976-30 2024-09-13T14:30:59.060 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 2 to smithi02683976-30 2024-09-13T14:30:59.061 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 3 to smithi02683976-30 2024-09-13T14:30:59.061 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 4 to smithi02683976-30 2024-09-13T14:30:59.062 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 5 to smithi02683976-30 2024-09-13T14:30:59.063 INFO:tasks.rados.rados.0.smithi026.stdout:3563: finishing write tid 6 to smithi02683976-30 2024-09-13T14:30:59.063 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1840 (ObjNum 1120 snap 337 seq_num 1120) dirty exists 2024-09-13T14:30:59.063 INFO:tasks.rados.rados.0.smithi026.stdout:3563: left oid 30 (ObjNum 1120 snap 337 seq_num 1120) 2024-09-13T14:31:00.027 INFO:tasks.rados.rados.0.smithi026.stdout:3558: done (5 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3560: done (4 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3561: done (3 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3562: done (2 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3563: done (1 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3564: done (0 left) 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3565: read oid 10 snap -1 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3565: expect deleted 2024-09-13T14:31:00.028 INFO:tasks.rados.rados.0.smithi026.stdout:3566: snap_create 2024-09-13T14:31:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: pgmap v724: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 677 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 5 op/s 2024-09-13T14:31:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:31:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: pgmap v724: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 677 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 5 op/s 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:00.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:00.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: pgmap v724: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 677 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 5 op/s 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:00.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:31:01.034 INFO:tasks.rados.rados.0.smithi026.stdout:3565: done (1 left) 2024-09-13T14:31:01.034 INFO:tasks.rados.rados.0.smithi026.stdout:3566: done (0 left) 2024-09-13T14:31:01.034 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 339 2024-09-13T14:31:01.034 INFO:tasks.rados.rados.0.smithi026.stdout:3567: seq_num 1121 ranges {3588096=778240,4366336=442368,4808704=638976,5447680=122880} 2024-09-13T14:31:01.058 INFO:tasks.rados.rados.0.smithi026.stdout:3567: writing smithi02683976-42 from 3588096 to 4366336 tid 1 2024-09-13T14:31:01.060 INFO:tasks.rados.rados.0.smithi026.stdout:3567: writing smithi02683976-42 from 4366336 to 4808704 tid 2 2024-09-13T14:31:01.064 INFO:tasks.rados.rados.0.smithi026.stdout:3567: writing smithi02683976-42 from 4808704 to 5447680 tid 3 2024-09-13T14:31:01.065 INFO:tasks.rados.rados.0.smithi026.stdout:3567: writing smithi02683976-42 from 5447680 to 5570560 tid 4 2024-09-13T14:31:01.065 INFO:tasks.rados.rados.0.smithi026.stdout:3568: setattr oid 12 current snap is 339 2024-09-13T14:31:01.065 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 1 to smithi02683976-42 2024-09-13T14:31:01.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 339 2024-09-13T14:31:01.066 INFO:tasks.rados.rados.0.smithi026.stdout:3569: seq_num 1122 ranges {6201344=794624,6995968=729088,7725056=475136} 2024-09-13T14:31:01.105 INFO:tasks.rados.rados.0.smithi026.stdout:3569: writing smithi02683976-25 from 6201344 to 6995968 tid 1 2024-09-13T14:31:01.108 INFO:tasks.rados.rados.0.smithi026.stdout:3569: writing smithi02683976-25 from 6995968 to 7725056 tid 2 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3569: writing smithi02683976-25 from 7725056 to 8200192 tid 3 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3570: read oid 23 snap -1 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3570: expect (ObjNum 1109 snap 335 seq_num 1109) 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3571: delete oid 17 current snap is 339 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 2 to smithi02683976-42 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 3 to smithi02683976-42 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 4 to smithi02683976-42 2024-09-13T14:31:01.112 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 5 to smithi02683976-42 2024-09-13T14:31:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:3567: finishing write tid 6 to smithi02683976-42 2024-09-13T14:31:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1479 (ObjNum 1121 snap 339 seq_num 1121) dirty exists 2024-09-13T14:31:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:3567: left oid 42 (ObjNum 1121 snap 339 seq_num 1121) 2024-09-13T14:31:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 2021 (ObjNum 65 snap 0 seq_num 1) dirty exists 2024-09-13T14:31:01.113 INFO:tasks.rados.rados.0.smithi026.stdout:3569: finishing write tid 1 to smithi02683976-25 2024-09-13T14:31:01.114 INFO:tasks.rados.rados.0.smithi026.stdout:3569: finishing write tid 2 to smithi02683976-25 2024-09-13T14:31:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:3569: finishing write tid 3 to smithi02683976-25 2024-09-13T14:31:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:3567: done (4 left) 2024-09-13T14:31:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:3568: done (3 left) 2024-09-13T14:31:01.115 INFO:tasks.rados.rados.0.smithi026.stdout:3571: done (2 left) 2024-09-13T14:31:01.116 INFO:tasks.rados.rados.0.smithi026.stdout:3572: setattr oid 46 current snap is 339 2024-09-13T14:31:01.116 INFO:tasks.rados.rados.0.smithi026.stdout:3569: finishing write tid 4 to smithi02683976-25 2024-09-13T14:31:01.117 INFO:tasks.rados.rados.0.smithi026.stdout:3569: finishing write tid 5 to smithi02683976-25 2024-09-13T14:31:01.117 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2015 (ObjNum 1122 snap 339 seq_num 1122) dirty exists 2024-09-13T14:31:01.117 INFO:tasks.rados.rados.0.smithi026.stdout:3569: left oid 25 (ObjNum 1122 snap 339 seq_num 1122) 2024-09-13T14:31:01.117 INFO:tasks.rados.rados.0.smithi026.stdout:3569: done (2 left) 2024-09-13T14:31:01.117 INFO:tasks.rados.rados.0.smithi026.stdout:3573: delete oid 4 current snap is 339 2024-09-13T14:31:01.121 INFO:tasks.rados.rados.0.smithi026.stdout:3573: done (2 left) 2024-09-13T14:31:01.121 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 339 2024-09-13T14:31:01.121 INFO:tasks.rados.rados.0.smithi026.stdout:3574: seq_num 1125 ranges {1548288=450560,1998848=786432,2785280=614400,3399680=204800} 2024-09-13T14:31:01.132 INFO:tasks.rados.rados.0.smithi026.stdout:3574: writing smithi02683976-33 from 1548288 to 1998848 tid 1 2024-09-13T14:31:01.136 INFO:tasks.rados.rados.0.smithi026.stdout:3574: writing smithi02683976-33 from 1998848 to 2785280 tid 2 2024-09-13T14:31:01.140 INFO:tasks.rados.rados.0.smithi026.stdout:3574: writing smithi02683976-33 from 2785280 to 3399680 tid 3 2024-09-13T14:31:01.141 INFO:tasks.rados.rados.0.smithi026.stdout:3574: writing smithi02683976-33 from 3399680 to 3604480 tid 4 2024-09-13T14:31:01.142 INFO:tasks.rados.rados.0.smithi026.stdout:3575: snap_remove snap 316 2024-09-13T14:31:01.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:01 smithi026 ceph-mon[97022]: osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:31:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:01 smithi026 ceph-mon[99461]: osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:31:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:01 smithi145 ceph-mon[89145]: osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:31:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:01] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.51.0" 2024-09-13T14:31:02.045 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1650 (ObjNum 1040 snap 319 seq_num 1040) dirty exists 2024-09-13T14:31:02.045 INFO:tasks.rados.rados.0.smithi026.stdout:3570: expect (ObjNum 1109 snap 335 seq_num 1109) 2024-09-13T14:31:02.162 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 1 to smithi02683976-33 2024-09-13T14:31:02.162 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 2 to smithi02683976-33 2024-09-13T14:31:02.162 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 3 to smithi02683976-33 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 4 to smithi02683976-33 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 5 to smithi02683976-33 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3574: finishing write tid 6 to smithi02683976-33 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2249 (ObjNum 1125 snap 339 seq_num 1125) dirty exists 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3574: left oid 33 (ObjNum 1125 snap 339 seq_num 1125) 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3570: done (3 left) 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3572: done (2 left) 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3574: done (1 left) 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3575: done (0 left) 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3576: copy_from oid 16 from oid 27 current snap is 339 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3577: rollback oid 21 current snap is 339 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 338 2024-09-13T14:31:02.163 INFO:tasks.rados.rados.0.smithi026.stdout:3578: snap_create 2024-09-13T14:31:02.164 INFO:tasks.rados.rados.0.smithi026.stdout:3576: finishing copy_from to smithi02683976-16 2024-09-13T14:31:02.164 INFO:tasks.rados.rados.0.smithi026.stdout:3576: got expected ENOENT (src dne) 2024-09-13T14:31:02.164 INFO:tasks.rados.rados.0.smithi026.stdout:3576: finishing copy_from racing read to smithi02683976-16 2024-09-13T14:31:02.166 INFO:tasks.rados.rados.0.smithi026.stdout:3577: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:31:02.166 INFO:tasks.rados.rados.0.smithi026.stdout:3577: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:31:02.166 INFO:tasks.rados.rados.0.smithi026.stdout:3577: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:31:02.167 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1819 (ObjNum 1104 snap 334 seq_num 1104) dirty exists 2024-09-13T14:31:02.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:02 smithi026 ceph-mon[99461]: pgmap v727: 121 pgs: 121 active+clean; 659 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 11 op/s 2024-09-13T14:31:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:02 smithi026 ceph-mon[99461]: osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:31:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:02 smithi026 ceph-mon[97022]: pgmap v727: 121 pgs: 121 active+clean; 659 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 11 op/s 2024-09-13T14:31:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:02 smithi026 ceph-mon[97022]: osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:31:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:02 smithi145 ceph-mon[89145]: pgmap v727: 121 pgs: 121 active+clean; 659 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.9 MiB/s wr, 11 op/s 2024-09-13T14:31:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:02 smithi145 ceph-mon[89145]: osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:31:03.045 INFO:tasks.rados.rados.0.smithi026.stdout:3576: done (2 left) 2024-09-13T14:31:03.045 INFO:tasks.rados.rados.0.smithi026.stdout:3577: done (1 left) 2024-09-13T14:31:03.045 INFO:tasks.rados.rados.0.smithi026.stdout:3578: done (0 left) 2024-09-13T14:31:03.045 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 4 current snap is 340 2024-09-13T14:31:03.045 INFO:tasks.rados.rados.0.smithi026.stdout:3579: seq_num 1126 ranges {0=466944,466944=720896,1187840=778240,1966080=106496} 2024-09-13T14:31:03.047 INFO:tasks.rados.rados.0.smithi026.stdout:3579: writing smithi02683976-4 from 0 to 466944 tid 1 2024-09-13T14:31:03.051 INFO:tasks.rados.rados.0.smithi026.stdout:3579: writing smithi02683976-4 from 466944 to 1187840 tid 2 2024-09-13T14:31:03.055 INFO:tasks.rados.rados.0.smithi026.stdout:3579: writing smithi02683976-4 from 1187840 to 1966080 tid 3 2024-09-13T14:31:03.056 INFO:tasks.rados.rados.0.smithi026.stdout:3579: writing smithi02683976-4 from 1966080 to 2072576 tid 4 2024-09-13T14:31:03.056 INFO:tasks.rados.rados.0.smithi026.stdout:3580: snap_remove snap 328 2024-09-13T14:31:03.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:03 smithi026 ceph-mon[97022]: osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:31:03.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:03 smithi026 ceph-mon[99461]: osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:31:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:03 smithi145 ceph-mon[89145]: osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:31:04.027 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 1 to smithi02683976-4 2024-09-13T14:31:04.027 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 2 to smithi02683976-4 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 3 to smithi02683976-4 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 4 to smithi02683976-4 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 5 to smithi02683976-4 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: finishing write tid 6 to smithi02683976-4 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 1846 (ObjNum 1126 snap 340 seq_num 1126) dirty exists 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: left oid 4 (ObjNum 1126 snap 340 seq_num 1126) 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3579: done (1 left) 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3580: done (0 left) 2024-09-13T14:31:04.028 INFO:tasks.rados.rados.0.smithi026.stdout:3581: snap_remove snap 322 2024-09-13T14:31:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[97022]: pgmap v730: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-09-13T14:31:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[97022]: osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:31:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[97022]: osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:31:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[99461]: pgmap v730: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-09-13T14:31:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[99461]: osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:31:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:04 smithi026 ceph-mon[99461]: osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:31:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:04 smithi145 ceph-mon[89145]: pgmap v730: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-09-13T14:31:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:04 smithi145 ceph-mon[89145]: osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:31:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:04 smithi145 ceph-mon[89145]: osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:31:05.028 INFO:tasks.rados.rados.0.smithi026.stdout:3581: done (0 left) 2024-09-13T14:31:05.028 INFO:tasks.rados.rados.0.smithi026.stdout:3582: delete oid 49 current snap is 340 2024-09-13T14:31:05.045 INFO:tasks.rados.rados.0.smithi026.stdout:3582: done (0 left) 2024-09-13T14:31:05.045 INFO:tasks.rados.rados.0.smithi026.stdout:3583: snap_remove snap 339 2024-09-13T14:31:06.050 INFO:tasks.rados.rados.0.smithi026.stdout:3583: done (0 left) 2024-09-13T14:31:06.050 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 340 2024-09-13T14:31:06.050 INFO:tasks.rados.rados.0.smithi026.stdout:3584: seq_num 1128 ranges {0=499712,499712=483328,983040=786432,1769472=335872} 2024-09-13T14:31:06.053 INFO:tasks.rados.rados.0.smithi026.stdout:3584: writing smithi02683976-50 from 0 to 499712 tid 1 2024-09-13T14:31:06.055 INFO:tasks.rados.rados.0.smithi026.stdout:3584: writing smithi02683976-50 from 499712 to 983040 tid 2 2024-09-13T14:31:06.060 INFO:tasks.rados.rados.0.smithi026.stdout:3584: writing smithi02683976-50 from 983040 to 1769472 tid 3 2024-09-13T14:31:06.062 INFO:tasks.rados.rados.0.smithi026.stdout:3584: writing smithi02683976-50 from 1769472 to 2105344 tid 4 2024-09-13T14:31:06.062 INFO:tasks.rados.rados.0.smithi026.stdout:3585: delete oid 42 current snap is 340 2024-09-13T14:31:06.062 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 1 to smithi02683976-50 2024-09-13T14:31:06.063 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 2 to smithi02683976-50 2024-09-13T14:31:06.064 INFO:tasks.rados.rados.0.smithi026.stdout:3585: done (1 left) 2024-09-13T14:31:06.064 INFO:tasks.rados.rados.0.smithi026.stdout:3586: snap_create 2024-09-13T14:31:06.066 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 3 to smithi02683976-50 2024-09-13T14:31:06.066 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 4 to smithi02683976-50 2024-09-13T14:31:06.067 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 5 to smithi02683976-50 2024-09-13T14:31:06.068 INFO:tasks.rados.rados.0.smithi026.stdout:3584: finishing write tid 6 to smithi02683976-50 2024-09-13T14:31:06.068 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 1836 (ObjNum 1128 snap 340 seq_num 1128) dirty exists 2024-09-13T14:31:06.068 INFO:tasks.rados.rados.0.smithi026.stdout:3584: left oid 50 (ObjNum 1128 snap 340 seq_num 1128) 2024-09-13T14:31:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:06 smithi026 ceph-mon[97022]: pgmap v733: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:31:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:06 smithi026 ceph-mon[97022]: osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:31:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:06 smithi026 ceph-mon[99461]: pgmap v733: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:31:06.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:06 smithi026 ceph-mon[99461]: osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:31:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:06 smithi145 ceph-mon[89145]: pgmap v733: 121 pgs: 121 active+clean; 667 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:31:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:06 smithi145 ceph-mon[89145]: osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:31:07.053 INFO:tasks.rados.rados.0.smithi026.stdout:3584: done (1 left) 2024-09-13T14:31:07.054 INFO:tasks.rados.rados.0.smithi026.stdout:3586: done (0 left) 2024-09-13T14:31:07.054 INFO:tasks.rados.rados.0.smithi026.stdout:3587: snap_create 2024-09-13T14:31:07.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:07 smithi026 ceph-mon[99461]: osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:31:07.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:07 smithi026 ceph-mon[97022]: osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:31:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:07 smithi145 ceph-mon[89145]: osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:3587: done (0 left) 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:3588: rollback oid 11 current snap is 342 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 338 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:3589: rollback oid 32 current snap is 342 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 32 to 335 2024-09-13T14:31:08.055 INFO:tasks.rados.rados.0.smithi026.stdout:3590: delete oid 15 current snap is 342 2024-09-13T14:31:08.058 INFO:tasks.rados.rados.0.smithi026.stdout:3589: finishing rollback tid 1 to smithi02683976-32 2024-09-13T14:31:08.058 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 1579 (ObjNum 65 snap 0 seq_num 1683383634) dirty dne 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:3589: done (2 left) 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:3590: done (1 left) 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:3591: rollback oid 21 current snap is 342 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 337 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 342 2024-09-13T14:31:08.065 INFO:tasks.rados.rados.0.smithi026.stdout:3592: seq_num 1131 ranges {0=548864,548864=540672,1089536=565248,1654784=425984,2080768=73728} 2024-09-13T14:31:08.068 INFO:tasks.rados.rados.0.smithi026.stdout:3592: writing smithi02683976-17 from 0 to 548864 tid 1 2024-09-13T14:31:08.071 INFO:tasks.rados.rados.0.smithi026.stdout:3592: writing smithi02683976-17 from 548864 to 1089536 tid 2 2024-09-13T14:31:08.074 INFO:tasks.rados.rados.0.smithi026.stdout:3592: writing smithi02683976-17 from 1089536 to 1654784 tid 3 2024-09-13T14:31:08.077 INFO:tasks.rados.rados.0.smithi026.stdout:3592: writing smithi02683976-17 from 1654784 to 2080768 tid 4 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3592: writing smithi02683976-17 from 2080768 to 2154496 tid 5 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3593: snap_create 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3588: finishing rollback tid 0 to smithi02683976-11 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3588: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3588: finishing rollback tid 2 to smithi02683976-11 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1840 (ObjNum 1048 snap 320 seq_num 1048) dirty exists 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3591: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:31:08.078 INFO:tasks.rados.rados.0.smithi026.stdout:3591: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:31:08.079 INFO:tasks.rados.rados.0.smithi026.stdout:3591: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:31:08.079 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1844 (ObjNum 1104 snap 334 seq_num 1104) dirty exists 2024-09-13T14:31:08.079 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 1 to smithi02683976-17 2024-09-13T14:31:08.079 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 2 to smithi02683976-17 2024-09-13T14:31:08.079 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 3 to smithi02683976-17 2024-09-13T14:31:08.080 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 4 to smithi02683976-17 2024-09-13T14:31:08.081 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 5 to smithi02683976-17 2024-09-13T14:31:08.081 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 6 to smithi02683976-17 2024-09-13T14:31:08.083 INFO:tasks.rados.rados.0.smithi026.stdout:3592: finishing write tid 7 to smithi02683976-17 2024-09-13T14:31:08.083 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1938 (ObjNum 1131 snap 342 seq_num 1131) dirty exists 2024-09-13T14:31:08.083 INFO:tasks.rados.rados.0.smithi026.stdout:3592: left oid 17 (ObjNum 1131 snap 342 seq_num 1131) 2024-09-13T14:31:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:08 smithi026 ceph-mon[97022]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:08 smithi026 ceph-mon[97022]: osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:31:08.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:08 smithi026 ceph-mon[99461]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:08 smithi026 ceph-mon[99461]: osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:31:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:08 smithi145 ceph-mon[89145]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:08 smithi145 ceph-mon[89145]: osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:3588: done (3 left) 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:3591: done (2 left) 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:3592: done (1 left) 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:3593: done (0 left) 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 343 2024-09-13T14:31:09.028 INFO:tasks.rados.rados.0.smithi026.stdout:3594: seq_num 1132 ranges {1884160=565248,2449408=425984,2875392=770048,3645440=409600} 2024-09-13T14:31:09.041 INFO:tasks.rados.rados.0.smithi026.stdout:3594: writing smithi02683976-13 from 1884160 to 2449408 tid 1 2024-09-13T14:31:09.043 INFO:tasks.rados.rados.0.smithi026.stdout:3594: writing smithi02683976-13 from 2449408 to 2875392 tid 2 2024-09-13T14:31:09.049 INFO:tasks.rados.rados.0.smithi026.stdout:3594: writing smithi02683976-13 from 2875392 to 3645440 tid 3 2024-09-13T14:31:09.052 INFO:tasks.rados.rados.0.smithi026.stdout:3594: writing smithi02683976-13 from 3645440 to 4055040 tid 4 2024-09-13T14:31:09.052 INFO:tasks.rados.rados.0.smithi026.stdout:3595: rollback oid 30 current snap is 343 2024-09-13T14:31:09.052 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 30 to 341 2024-09-13T14:31:09.052 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 1 to smithi02683976-13 2024-09-13T14:31:09.052 INFO:tasks.rados.rados.0.smithi026.stdout:3596: snap_create 2024-09-13T14:31:09.053 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 2 to smithi02683976-13 2024-09-13T14:31:09.056 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 3 to smithi02683976-13 2024-09-13T14:31:09.057 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 4 to smithi02683976-13 2024-09-13T14:31:09.057 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 5 to smithi02683976-13 2024-09-13T14:31:09.058 INFO:tasks.rados.rados.0.smithi026.stdout:3595: finishing rollback tid 0 to smithi02683976-30 2024-09-13T14:31:09.058 INFO:tasks.rados.rados.0.smithi026.stdout:3595: finishing rollback tid 1 to smithi02683976-30 2024-09-13T14:31:09.059 INFO:tasks.rados.rados.0.smithi026.stdout:3595: finishing rollback tid 2 to smithi02683976-30 2024-09-13T14:31:09.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1850 (ObjNum 1120 snap 337 seq_num 1120) dirty exists 2024-09-13T14:31:09.059 INFO:tasks.rados.rados.0.smithi026.stdout:3594: finishing write tid 6 to smithi02683976-13 2024-09-13T14:31:09.059 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 2039 (ObjNum 1132 snap 343 seq_num 1132) dirty exists 2024-09-13T14:31:09.059 INFO:tasks.rados.rados.0.smithi026.stdout:3594: left oid 13 (ObjNum 1132 snap 343 seq_num 1132) 2024-09-13T14:31:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:09 smithi026 ceph-mon[97022]: osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:31:09.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:09 smithi026 ceph-mon[97022]: osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:31:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:09 smithi026 ceph-mon[99461]: osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:31:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:09 smithi026 ceph-mon[99461]: osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:31:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:09 smithi145 ceph-mon[89145]: osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:31:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:09 smithi145 ceph-mon[89145]: osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:31:10.029 INFO:tasks.rados.rados.0.smithi026.stdout:3594: done (2 left) 2024-09-13T14:31:10.029 INFO:tasks.rados.rados.0.smithi026.stdout:3595: done (1 left) 2024-09-13T14:31:10.029 INFO:tasks.rados.rados.0.smithi026.stdout:3596: done (0 left) 2024-09-13T14:31:10.030 INFO:tasks.rados.rados.0.smithi026.stdout:3597: read oid 41 snap -1 2024-09-13T14:31:10.030 INFO:tasks.rados.rados.0.smithi026.stdout:3597: expect (ObjNum 949 snap 296 seq_num 949) 2024-09-13T14:31:10.030 INFO:tasks.rados.rados.0.smithi026.stdout:3598: snap_remove snap 336 2024-09-13T14:31:10.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:10 smithi026 ceph-mon[99461]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:10.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:10 smithi026 ceph-mon[99461]: osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:31:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:10 smithi026 ceph-mon[97022]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:10 smithi026 ceph-mon[97022]: osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:31:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:10 smithi145 ceph-mon[89145]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 118 active+clean; 645 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-13T14:31:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:10 smithi145 ceph-mon[89145]: osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:31:10.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:11.035 INFO:tasks.rados.rados.0.smithi026.stdout:3597: expect (ObjNum 949 snap 296 seq_num 949) 2024-09-13T14:31:11.235 INFO:tasks.rados.rados.0.smithi026.stdout:3597: done (1 left) 2024-09-13T14:31:11.242 INFO:tasks.rados.rados.0.smithi026.stdout:3598: done (0 left) 2024-09-13T14:31:11.242 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 41 current snap is 344 2024-09-13T14:31:11.243 INFO:tasks.rados.rados.0.smithi026.stdout:3599: seq_num 1133 ranges {4628480=581632,5210112=753664,5963776=573440,6537216=278528} 2024-09-13T14:31:11.263 INFO:tasks.rados.rados.0.smithi026.stdout:3599: writing smithi02683976-41 from 4628480 to 5210112 tid 1 2024-09-13T14:31:11.267 INFO:tasks.rados.rados.0.smithi026.stdout:3599: writing smithi02683976-41 from 5210112 to 5963776 tid 2 2024-09-13T14:31:11.270 INFO:tasks.rados.rados.0.smithi026.stdout:3599: writing smithi02683976-41 from 5963776 to 6537216 tid 3 2024-09-13T14:31:11.272 INFO:tasks.rados.rados.0.smithi026.stdout:3599: writing smithi02683976-41 from 6537216 to 6815744 tid 4 2024-09-13T14:31:11.272 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 344 2024-09-13T14:31:11.272 INFO:tasks.rados.rados.0.smithi026.stdout:3600: seq_num 1134 ranges {0=598016,598016=638976,1236992=737280,1974272=229376} 2024-09-13T14:31:11.275 INFO:tasks.rados.rados.0.smithi026.stdout:3600: writing smithi02683976-27 from 0 to 598016 tid 1 2024-09-13T14:31:11.278 INFO:tasks.rados.rados.0.smithi026.stdout:3600: writing smithi02683976-27 from 598016 to 1236992 tid 2 2024-09-13T14:31:11.283 INFO:tasks.rados.rados.0.smithi026.stdout:3600: writing smithi02683976-27 from 1236992 to 1974272 tid 3 2024-09-13T14:31:11.285 INFO:tasks.rados.rados.0.smithi026.stdout:3600: writing smithi02683976-27 from 1974272 to 2203648 tid 4 2024-09-13T14:31:11.285 INFO:tasks.rados.rados.0.smithi026.stdout:3601: read oid 3 snap -1 2024-09-13T14:31:11.285 INFO:tasks.rados.rados.0.smithi026.stdout:3601: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:31:11.285 INFO:tasks.rados.rados.0.smithi026.stdout:3602: snap_remove snap 332 2024-09-13T14:31:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:11] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:31:12.070 INFO:tasks.rados.rados.0.smithi026.stdout:3602: done (3 left) 2024-09-13T14:31:12.071 INFO:tasks.rados.rados.0.smithi026.stdout:3603: read oid 20 snap -1 2024-09-13T14:31:12.071 INFO:tasks.rados.rados.0.smithi026.stdout:3603: expect deleted 2024-09-13T14:31:12.071 INFO:tasks.rados.rados.0.smithi026.stdout:3604: rmattr oid 17 current snap is 344 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 1 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 2 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 3 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 4 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 5 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: finishing write tid 6 to smithi02683976-41 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 41 v 1943 (ObjNum 1133 snap 344 seq_num 1133) dirty exists 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3599: left oid 41 (ObjNum 1133 snap 344 seq_num 1133) 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 1 to smithi02683976-27 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 2 to smithi02683976-27 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 3 to smithi02683976-27 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 4 to smithi02683976-27 2024-09-13T14:31:12.073 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 5 to smithi02683976-27 2024-09-13T14:31:12.074 INFO:tasks.rados.rados.0.smithi026.stdout:3600: finishing write tid 6 to smithi02683976-27 2024-09-13T14:31:12.074 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2290 (ObjNum 1134 snap 344 seq_num 1134) dirty exists 2024-09-13T14:31:12.074 INFO:tasks.rados.rados.0.smithi026.stdout:3600: left oid 27 (ObjNum 1134 snap 344 seq_num 1134) 2024-09-13T14:31:12.074 INFO:tasks.rados.rados.0.smithi026.stdout:3601: expect (ObjNum 931 snap 290 seq_num 931) 2024-09-13T14:31:12.241 INFO:tasks.rados.rados.0.smithi026.stdout:3599: done (4 left) 2024-09-13T14:31:12.241 INFO:tasks.rados.rados.0.smithi026.stdout:3600: done (3 left) 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3601: done (2 left) 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3603: done (1 left) 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3604: done (0 left) 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3605: read oid 16 snap 330 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3605: expect (ObjNum 1040 snap 319 seq_num 1040) 2024-09-13T14:31:12.242 INFO:tasks.rados.rados.0.smithi026.stdout:3606: delete oid 11 current snap is 344 2024-09-13T14:31:12.244 INFO:tasks.rados.rados.0.smithi026.stdout:3606: done (1 left) 2024-09-13T14:31:12.244 INFO:tasks.rados.rados.0.smithi026.stdout:3607: snap_remove snap 340 2024-09-13T14:31:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[99461]: pgmap v742: 121 pgs: 121 active+clean; 632 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-13T14:31:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[99461]: osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:31:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[97022]: pgmap v742: 121 pgs: 121 active+clean; 632 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-13T14:31:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[97022]: osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:31:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:12 smithi145 ceph-mon[89145]: pgmap v742: 121 pgs: 121 active+clean; 632 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-13T14:31:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:12 smithi145 ceph-mon[89145]: osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:31:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:13.076 INFO:tasks.rados.rados.0.smithi026.stdout:3607: done (1 left) 2024-09-13T14:31:13.076 INFO:tasks.rados.rados.0.smithi026.stdout:3608: read oid 20 snap -1 2024-09-13T14:31:13.076 INFO:tasks.rados.rados.0.smithi026.stdout:3608: expect deleted 2024-09-13T14:31:13.076 INFO:tasks.rados.rados.0.smithi026.stdout:3609: snap_create 2024-09-13T14:31:13.076 INFO:tasks.rados.rados.0.smithi026.stdout:3605: expect (ObjNum 1040 snap 319 seq_num 1040) 2024-09-13T14:31:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:13 smithi026 ceph-mon[97022]: osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:31:13.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:13 smithi026 ceph-mon[99461]: osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:31:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:13 smithi145 ceph-mon[89145]: osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3605: done (2 left) 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3608: done (1 left) 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3609: done (0 left) 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3610: read oid 48 snap -1 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3610: expect (ObjNum 1050 snap 320 seq_num 1050) 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 345 2024-09-13T14:31:14.030 INFO:tasks.rados.rados.0.smithi026.stdout:3611: seq_num 1136 ranges {1736704=630784,2367488=409600,2777088=753664,3530752=417792,3948544=24576} 2024-09-13T14:31:14.043 INFO:tasks.rados.rados.0.smithi026.stdout:3611: writing smithi02683976-31 from 1736704 to 2367488 tid 1 2024-09-13T14:31:14.045 INFO:tasks.rados.rados.0.smithi026.stdout:3611: writing smithi02683976-31 from 2367488 to 2777088 tid 2 2024-09-13T14:31:14.052 INFO:tasks.rados.rados.0.smithi026.stdout:3611: writing smithi02683976-31 from 2777088 to 3530752 tid 3 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:3611: writing smithi02683976-31 from 3530752 to 3948544 tid 4 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:3611: writing smithi02683976-31 from 3948544 to 3973120 tid 5 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:3612: read oid 49 snap -1 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:3612: expect deleted 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 345 2024-09-13T14:31:14.053 INFO:tasks.rados.rados.0.smithi026.stdout:3613: seq_num 1137 ranges {0=647168,647168=696320,1343488=557056,1900544=352256} 2024-09-13T14:31:14.056 INFO:tasks.rados.rados.0.smithi026.stdout:3613: writing smithi02683976-10 from 0 to 647168 tid 1 2024-09-13T14:31:14.060 INFO:tasks.rados.rados.0.smithi026.stdout:3613: writing smithi02683976-10 from 647168 to 1343488 tid 2 2024-09-13T14:31:14.064 INFO:tasks.rados.rados.0.smithi026.stdout:3613: writing smithi02683976-10 from 1343488 to 1900544 tid 3 2024-09-13T14:31:14.066 INFO:tasks.rados.rados.0.smithi026.stdout:3613: writing smithi02683976-10 from 1900544 to 2252800 tid 4 2024-09-13T14:31:14.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 345 2024-09-13T14:31:14.066 INFO:tasks.rados.rados.0.smithi026.stdout:3610: expect (ObjNum 1050 snap 320 seq_num 1050) 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 1 to smithi02683976-31 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 2 to smithi02683976-31 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 3 to smithi02683976-31 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 4 to smithi02683976-31 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 5 to smithi02683976-31 2024-09-13T14:31:14.089 INFO:tasks.rados.rados.0.smithi026.stdout:3614: seq_num 1138 ranges {2351104=663552,3014656=581632,3596288=712704,4308992=311296} 2024-09-13T14:31:14.105 INFO:tasks.rados.rados.0.smithi026.stdout:3614: writing smithi02683976-7 from 2351104 to 3014656 tid 1 2024-09-13T14:31:14.108 INFO:tasks.rados.rados.0.smithi026.stdout:3614: writing smithi02683976-7 from 3014656 to 3596288 tid 2 2024-09-13T14:31:14.113 INFO:tasks.rados.rados.0.smithi026.stdout:3614: writing smithi02683976-7 from 3596288 to 4308992 tid 3 2024-09-13T14:31:14.115 INFO:tasks.rados.rados.0.smithi026.stdout:3614: writing smithi02683976-7 from 4308992 to 4620288 tid 4 2024-09-13T14:31:14.115 INFO:tasks.rados.rados.0.smithi026.stdout:3610: done (4 left) 2024-09-13T14:31:14.115 INFO:tasks.rados.rados.0.smithi026.stdout:3612: done (3 left) 2024-09-13T14:31:14.115 INFO:tasks.rados.rados.0.smithi026.stdout:3615: read oid 18 snap -1 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3615: expect (ObjNum 1100 snap 334 seq_num 1100) 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3616: snap_create 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 6 to smithi02683976-31 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3611: finishing write tid 7 to smithi02683976-31 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1871 (ObjNum 1136 snap 345 seq_num 1136) dirty exists 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3611: left oid 31 (ObjNum 1136 snap 345 seq_num 1136) 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 1 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 2 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 3 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 4 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 5 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:3613: finishing write tid 6 to smithi02683976-10 2024-09-13T14:31:14.116 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 2028 (ObjNum 1137 snap 345 seq_num 1137) dirty exists 2024-09-13T14:31:14.117 INFO:tasks.rados.rados.0.smithi026.stdout:3613: left oid 10 (ObjNum 1137 snap 345 seq_num 1137) 2024-09-13T14:31:14.117 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 1 to smithi02683976-7 2024-09-13T14:31:14.117 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 2 to smithi02683976-7 2024-09-13T14:31:14.117 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 3 to smithi02683976-7 2024-09-13T14:31:14.117 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 4 to smithi02683976-7 2024-09-13T14:31:14.118 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 5 to smithi02683976-7 2024-09-13T14:31:14.119 INFO:tasks.rados.rados.0.smithi026.stdout:3614: finishing write tid 6 to smithi02683976-7 2024-09-13T14:31:14.120 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1613 (ObjNum 1138 snap 345 seq_num 1138) dirty exists 2024-09-13T14:31:14.120 INFO:tasks.rados.rados.0.smithi026.stdout:3614: left oid 7 (ObjNum 1138 snap 345 seq_num 1138) 2024-09-13T14:31:14.126 INFO:tasks.rados.rados.0.smithi026.stdout:3615: expect (ObjNum 1100 snap 334 seq_num 1100) 2024-09-13T14:31:14.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[97022]: pgmap v745: 121 pgs: 121 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-13T14:31:14.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[97022]: osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:31:14.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[97022]: osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:31:14.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[99461]: pgmap v745: 121 pgs: 121 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-13T14:31:14.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[99461]: osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:31:14.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:14 smithi026 ceph-mon[99461]: osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:31:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:14 smithi145 ceph-mon[89145]: pgmap v745: 121 pgs: 121 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-13T14:31:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:14 smithi145 ceph-mon[89145]: osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:31:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:14 smithi145 ceph-mon[89145]: osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:31:15.081 INFO:tasks.rados.rados.0.smithi026.stdout:3611: done (4 left) 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3613: done (3 left) 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3614: done (2 left) 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3615: done (1 left) 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3616: done (0 left) 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3617: copy_from oid 23 from oid 31 current snap is 346 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3618: read oid 11 snap -1 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3618: expect deleted 2024-09-13T14:31:15.082 INFO:tasks.rados.rados.0.smithi026.stdout:3619: snap_create 2024-09-13T14:31:15.105 INFO:tasks.rados.rados.0.smithi026.stdout:3617: finishing copy_from racing read to smithi02683976-23 2024-09-13T14:31:15.108 INFO:tasks.rados.rados.0.smithi026.stdout:3617: finishing copy_from to smithi02683976-23 2024-09-13T14:31:15.108 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 1871 (ObjNum 1136 snap 345 seq_num 1136) dirty exists 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3617: done (2 left) 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3618: done (1 left) 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3619: done (0 left) 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3620: rollback oid 45 current snap is 347 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 333 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 46 current snap is 347 2024-09-13T14:31:16.084 INFO:tasks.rados.rados.0.smithi026.stdout:3621: seq_num 1139 ranges {3817472=679936,4497408=458752,4956160=524288,5480448=540672,6021120=81920} 2024-09-13T14:31:16.108 INFO:tasks.rados.rados.0.smithi026.stdout:3621: writing smithi02683976-46 from 3817472 to 4497408 tid 1 2024-09-13T14:31:16.110 INFO:tasks.rados.rados.0.smithi026.stdout:3621: writing smithi02683976-46 from 4497408 to 4956160 tid 2 2024-09-13T14:31:16.114 INFO:tasks.rados.rados.0.smithi026.stdout:3621: writing smithi02683976-46 from 4956160 to 5480448 tid 3 2024-09-13T14:31:16.117 INFO:tasks.rados.rados.0.smithi026.stdout:3621: writing smithi02683976-46 from 5480448 to 6021120 tid 4 2024-09-13T14:31:16.118 INFO:tasks.rados.rados.0.smithi026.stdout:3621: writing smithi02683976-46 from 6021120 to 6103040 tid 5 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3622: rollback oid 12 current snap is 347 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 345 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3620: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 347 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3620: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3620: finishing rollback tid 2 to smithi02683976-45 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1207 (ObjNum 1054 snap 323 seq_num 1054) dirty exists 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 1 to smithi02683976-46 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 2 to smithi02683976-46 2024-09-13T14:31:16.119 INFO:tasks.rados.rados.0.smithi026.stdout:3623: seq_num 1140 ranges {15163392=696320,15859712=745472,16605184=729088,17334272=131072} 2024-09-13T14:31:16.205 INFO:tasks.rados.rados.0.smithi026.stdout:3623: writing smithi02683976-22 from 15163392 to 15859712 tid 1 2024-09-13T14:31:16.209 INFO:tasks.rados.rados.0.smithi026.stdout:3623: writing smithi02683976-22 from 15859712 to 16605184 tid 2 2024-09-13T14:31:16.213 INFO:tasks.rados.rados.0.smithi026.stdout:3623: writing smithi02683976-22 from 16605184 to 17334272 tid 3 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3623: writing smithi02683976-22 from 17334272 to 17465344 tid 4 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3620: done (3 left) 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3624: snap_create 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 3 to smithi02683976-46 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 4 to smithi02683976-46 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 5 to smithi02683976-46 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 6 to smithi02683976-46 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3622: finishing rollback tid 0 to smithi02683976-12 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3622: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:31:16.214 INFO:tasks.rados.rados.0.smithi026.stdout:3622: finishing rollback tid 2 to smithi02683976-12 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 2024 (ObjNum 65 snap 0 seq_num 1886219632) dirty exists 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:3621: finishing write tid 7 to smithi02683976-46 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1656 (ObjNum 1139 snap 347 seq_num 1139) dirty exists 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:3621: left oid 46 (ObjNum 1139 snap 347 seq_num 1139) 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 1 to smithi02683976-22 2024-09-13T14:31:16.215 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 2 to smithi02683976-22 2024-09-13T14:31:16.219 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 3 to smithi02683976-22 2024-09-13T14:31:16.219 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 4 to smithi02683976-22 2024-09-13T14:31:16.219 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 5 to smithi02683976-22 2024-09-13T14:31:16.221 INFO:tasks.rados.rados.0.smithi026.stdout:3623: finishing write tid 6 to smithi02683976-22 2024-09-13T14:31:16.221 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2029 (ObjNum 1140 snap 347 seq_num 1140) dirty exists 2024-09-13T14:31:16.221 INFO:tasks.rados.rados.0.smithi026.stdout:3623: left oid 22 (ObjNum 1140 snap 347 seq_num 1140) 2024-09-13T14:31:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:16 smithi026 ceph-mon[97022]: pgmap v748: 121 pgs: 121 active+clean; 635 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 784 KiB/s wr, 2 op/s 2024-09-13T14:31:16.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:16 smithi026 ceph-mon[97022]: osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:31:16.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:16 smithi026 ceph-mon[99461]: pgmap v748: 121 pgs: 121 active+clean; 635 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 784 KiB/s wr, 2 op/s 2024-09-13T14:31:16.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:16 smithi026 ceph-mon[99461]: osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:31:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:16 smithi145 ceph-mon[89145]: pgmap v748: 121 pgs: 121 active+clean; 635 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 784 KiB/s wr, 2 op/s 2024-09-13T14:31:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:16 smithi145 ceph-mon[89145]: osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:31:17.094 INFO:tasks.rados.rados.0.smithi026.stdout:3621: done (3 left) 2024-09-13T14:31:17.094 INFO:tasks.rados.rados.0.smithi026.stdout:3622: done (2 left) 2024-09-13T14:31:17.094 INFO:tasks.rados.rados.0.smithi026.stdout:3623: done (1 left) 2024-09-13T14:31:17.095 INFO:tasks.rados.rados.0.smithi026.stdout:3624: done (0 left) 2024-09-13T14:31:17.095 INFO:tasks.rados.rados.0.smithi026.stdout:3625: delete oid 49 current snap is 348 2024-09-13T14:31:17.096 INFO:tasks.rados.rados.0.smithi026.stdout:3625: done (0 left) 2024-09-13T14:31:17.097 INFO:tasks.rados.rados.0.smithi026.stdout:3626: snap_remove snap 291 2024-09-13T14:31:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:17 smithi026 ceph-mon[97022]: osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:31:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:17 smithi026 ceph-mon[99461]: osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:31:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:17 smithi145 ceph-mon[89145]: osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:31:18.099 INFO:tasks.rados.rados.0.smithi026.stdout:3626: done (0 left) 2024-09-13T14:31:18.099 INFO:tasks.rados.rados.0.smithi026.stdout:3627: rollback oid 20 current snap is 348 2024-09-13T14:31:18.100 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 256 2024-09-13T14:31:18.100 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 10 current snap is 348 2024-09-13T14:31:18.100 INFO:tasks.rados.rados.0.smithi026.stdout:3628: seq_num 1142 ranges {2252800=729088,2981888=565248,3547136=696320,4243456=344064} 2024-09-13T14:31:18.118 INFO:tasks.rados.rados.0.smithi026.stdout:3628: writing smithi02683976-10 from 2252800 to 2981888 tid 1 2024-09-13T14:31:18.121 INFO:tasks.rados.rados.0.smithi026.stdout:3628: writing smithi02683976-10 from 2981888 to 3547136 tid 2 2024-09-13T14:31:18.127 INFO:tasks.rados.rados.0.smithi026.stdout:3628: writing smithi02683976-10 from 3547136 to 4243456 tid 3 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:3628: writing smithi02683976-10 from 4243456 to 4587520 tid 4 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:3629: copy_from oid 24 from oid 49 current snap is 348 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:3630: rollback oid 36 current snap is 348 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 347 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:3627: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 0 (ObjNum 5873 snap 0 seq_num 538562592) dirty dne 2024-09-13T14:31:18.131 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 1 to smithi02683976-10 2024-09-13T14:31:18.132 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 2 to smithi02683976-10 2024-09-13T14:31:18.132 INFO:tasks.rados.rados.0.smithi026.stdout:3627: done (3 left) 2024-09-13T14:31:18.132 INFO:tasks.rados.rados.0.smithi026.stdout:3631: snap_remove snap 300 2024-09-13T14:31:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:18 smithi026 ceph-mon[97022]: pgmap v751: 121 pgs: 121 active+clean; 628 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:31:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:18 smithi026 ceph-mon[97022]: osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:31:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:18 smithi026 ceph-mon[99461]: pgmap v751: 121 pgs: 121 active+clean; 628 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:31:18.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:18 smithi026 ceph-mon[99461]: osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:31:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:18 smithi145 ceph-mon[89145]: pgmap v751: 121 pgs: 121 active+clean; 628 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-13T14:31:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:18 smithi145 ceph-mon[89145]: osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3630: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3630: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 3 to smithi02683976-10 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 4 to smithi02683976-10 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 5 to smithi02683976-10 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3629: finishing copy_from to smithi02683976-24 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3629: got expected ENOENT (src dne) 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3629: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3628: finishing write tid 6 to smithi02683976-10 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 2033 (ObjNum 1142 snap 348 seq_num 1142) dirty exists 2024-09-13T14:31:19.032 INFO:tasks.rados.rados.0.smithi026.stdout:3628: left oid 10 (ObjNum 1142 snap 348 seq_num 1142) 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3630: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2314 (ObjNum 1098 snap 334 seq_num 1098) dirty exists 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3628: done (3 left) 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3629: done (2 left) 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3630: done (1 left) 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3631: done (0 left) 2024-09-13T14:31:19.033 INFO:tasks.rados.rados.0.smithi026.stdout:3632: delete oid 8 current snap is 348 2024-09-13T14:31:19.044 INFO:tasks.rados.rados.0.smithi026.stdout:3632: done (0 left) 2024-09-13T14:31:19.044 INFO:tasks.rados.rados.0.smithi026.stdout:3633: read oid 27 snap -1 2024-09-13T14:31:19.044 INFO:tasks.rados.rados.0.smithi026.stdout:3633: expect (ObjNum 1134 snap 344 seq_num 1134) 2024-09-13T14:31:19.044 INFO:tasks.rados.rados.0.smithi026.stdout:3634: copy_from oid 25 from oid 17 current snap is 348 2024-09-13T14:31:19.045 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 348 2024-09-13T14:31:19.045 INFO:tasks.rados.rados.0.smithi026.stdout:3635: seq_num 1144 ranges {3784704=770048,4554752=737280,5292032=712704,6004736=155648} 2024-09-13T14:31:19.071 INFO:tasks.rados.rados.0.smithi026.stdout:3635: writing smithi02683976-19 from 3784704 to 4554752 tid 1 2024-09-13T14:31:19.076 INFO:tasks.rados.rados.0.smithi026.stdout:3635: writing smithi02683976-19 from 4554752 to 5292032 tid 2 2024-09-13T14:31:19.080 INFO:tasks.rados.rados.0.smithi026.stdout:3635: writing smithi02683976-19 from 5292032 to 6004736 tid 3 2024-09-13T14:31:19.082 INFO:tasks.rados.rados.0.smithi026.stdout:3635: writing smithi02683976-19 from 6004736 to 6160384 tid 4 2024-09-13T14:31:19.082 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 42 current snap is 348 2024-09-13T14:31:19.082 INFO:tasks.rados.rados.0.smithi026.stdout:3636: seq_num 1145 ranges {0=786432,786432=622592,1409024=516096,1925120=417792,2342912=49152} 2024-09-13T14:31:19.087 INFO:tasks.rados.rados.0.smithi026.stdout:3636: writing smithi02683976-42 from 0 to 786432 tid 1 2024-09-13T14:31:19.091 INFO:tasks.rados.rados.0.smithi026.stdout:3636: writing smithi02683976-42 from 786432 to 1409024 tid 2 2024-09-13T14:31:19.094 INFO:tasks.rados.rados.0.smithi026.stdout:3636: writing smithi02683976-42 from 1409024 to 1925120 tid 3 2024-09-13T14:31:19.096 INFO:tasks.rados.rados.0.smithi026.stdout:3636: writing smithi02683976-42 from 1925120 to 2342912 tid 4 2024-09-13T14:31:19.097 INFO:tasks.rados.rados.0.smithi026.stdout:3636: writing smithi02683976-42 from 2342912 to 2392064 tid 5 2024-09-13T14:31:19.097 INFO:tasks.rados.rados.0.smithi026.stdout:3637: rollback oid 10 current snap is 348 2024-09-13T14:31:19.097 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 338 2024-09-13T14:31:19.098 INFO:tasks.rados.rados.0.smithi026.stdout:3638: read oid 26 snap -1 2024-09-13T14:31:19.098 INFO:tasks.rados.rados.0.smithi026.stdout:3638: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:31:19.098 INFO:tasks.rados.rados.0.smithi026.stdout:3639: snap_remove snap 256 2024-09-13T14:31:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:19 smithi026 ceph-mon[97022]: osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:31:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:19 smithi026 ceph-mon[97022]: osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:31:19.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:19 smithi026 ceph-mon[99461]: osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:31:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:19 smithi026 ceph-mon[99461]: osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:31:19.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:19 smithi145 ceph-mon[89145]: osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:31:19.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:19 smithi145 ceph-mon[89145]: osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:31:20.104 INFO:tasks.rados.rados.0.smithi026.stdout:3639: done (6 left) 2024-09-13T14:31:20.104 INFO:tasks.rados.rados.0.smithi026.stdout:3640: read oid 14 snap -1 2024-09-13T14:31:20.104 INFO:tasks.rados.rados.0.smithi026.stdout:3640: expect deleted 2024-09-13T14:31:20.104 INFO:tasks.rados.rados.0.smithi026.stdout:3641: read oid 49 snap -1 2024-09-13T14:31:20.105 INFO:tasks.rados.rados.0.smithi026.stdout:3641: expect deleted 2024-09-13T14:31:20.105 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 6 current snap is 348 2024-09-13T14:31:20.105 INFO:tasks.rados.rados.0.smithi026.stdout:3642: seq_num 1146 ranges {606208=802816,1409024=499712,1908736=679936,2588672=425984} 2024-09-13T14:31:20.113 INFO:tasks.rados.rados.0.smithi026.stdout:3642: writing smithi02683976-6 from 606208 to 1409024 tid 1 2024-09-13T14:31:20.117 INFO:tasks.rados.rados.0.smithi026.stdout:3642: writing smithi02683976-6 from 1409024 to 1908736 tid 2 2024-09-13T14:31:20.122 INFO:tasks.rados.rados.0.smithi026.stdout:3642: writing smithi02683976-6 from 1908736 to 2588672 tid 3 2024-09-13T14:31:20.125 INFO:tasks.rados.rados.0.smithi026.stdout:3642: writing smithi02683976-6 from 2588672 to 3014656 tid 4 2024-09-13T14:31:20.126 INFO:tasks.rados.rados.0.smithi026.stdout:3643: delete oid 35 current snap is 348 2024-09-13T14:31:20.126 INFO:tasks.rados.rados.0.smithi026.stdout:3633: expect (ObjNum 1134 snap 344 seq_num 1134) 2024-09-13T14:31:20.192 INFO:tasks.rados.rados.0.smithi026.stdout:3634: finishing copy_from racing read to smithi02683976-25 2024-09-13T14:31:20.192 INFO:tasks.rados.rados.0.smithi026.stdout:3634: finishing copy_from to smithi02683976-25 2024-09-13T14:31:20.192 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 1938 (ObjNum 1131 snap 342 seq_num 1131) dirty exists 2024-09-13T14:31:20.192 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 1 to smithi02683976-19 2024-09-13T14:31:20.192 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 2 to smithi02683976-19 2024-09-13T14:31:20.193 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 3 to smithi02683976-19 2024-09-13T14:31:20.193 INFO:tasks.rados.rados.0.smithi026.stdout:3633: done (9 left) 2024-09-13T14:31:20.193 INFO:tasks.rados.rados.0.smithi026.stdout:3634: done (8 left) 2024-09-13T14:31:20.193 INFO:tasks.rados.rados.0.smithi026.stdout:3643: done (7 left) 2024-09-13T14:31:20.193 INFO:tasks.rados.rados.0.smithi026.stdout:3644: snap_remove snap 330 2024-09-13T14:31:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:20 smithi026 ceph-mon[97022]: pgmap v754: 121 pgs: 121 active+clean; 631 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-13T14:31:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:20 smithi026 ceph-mon[99461]: pgmap v754: 121 pgs: 121 active+clean; 631 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-13T14:31:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:20 smithi145 ceph-mon[89145]: pgmap v754: 121 pgs: 121 active+clean; 631 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-13T14:31:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3644: done (7 left) 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3645: rollback oid 25 current snap is 348 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 343 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 4 to smithi02683976-19 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 5 to smithi02683976-19 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3635: finishing write tid 6 to smithi02683976-19 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2044 (ObjNum 1144 snap 348 seq_num 1144) dirty exists 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3635: left oid 19 (ObjNum 1144 snap 348 seq_num 1144) 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 1 to smithi02683976-42 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 2 to smithi02683976-42 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 3 to smithi02683976-42 2024-09-13T14:31:21.119 INFO:tasks.rados.rados.0.smithi026.stdout:3637: finishing rollback tid 0 to smithi02683976-10 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 4 to smithi02683976-42 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 5 to smithi02683976-42 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3637: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 2035 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 6 to smithi02683976-42 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3636: finishing write tid 7 to smithi02683976-42 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 42 v 1486 (ObjNum 1145 snap 348 seq_num 1145) dirty exists 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3636: left oid 42 (ObjNum 1145 snap 348 seq_num 1145) 2024-09-13T14:31:21.120 INFO:tasks.rados.rados.0.smithi026.stdout:3638: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:31:21.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:21 smithi026 ceph-mon[99461]: osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:31:21.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:21 smithi026 ceph-mon[97022]: osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:31:21.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:21] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:31:21.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:21 smithi145 ceph-mon[89145]: osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3635: done (7 left) 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3636: done (6 left) 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3637: done (5 left) 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3638: done (4 left) 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3640: done (3 left) 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3646: delete oid 40 current snap is 348 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 1 to smithi02683976-6 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 2 to smithi02683976-6 2024-09-13T14:31:22.311 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 3 to smithi02683976-6 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 4 to smithi02683976-6 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 5 to smithi02683976-6 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3642: finishing write tid 6 to smithi02683976-6 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1897 (ObjNum 1146 snap 348 seq_num 1146) dirty exists 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3642: left oid 6 (ObjNum 1146 snap 348 seq_num 1146) 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3645: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3645: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:3645: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:31:22.312 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2018 (ObjNum 1122 snap 339 seq_num 1122) dirty exists 2024-09-13T14:31:22.313 INFO:tasks.rados.rados.0.smithi026.stdout:3641: done (3 left) 2024-09-13T14:31:22.313 INFO:tasks.rados.rados.0.smithi026.stdout:3642: done (2 left) 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3645: done (1 left) 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3646: done (0 left) 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3647: read oid 22 snap 334 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3647: expect (ObjNum 1099 snap 334 seq_num 1099) 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3648: read oid 1 snap -1 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3648: expect (ObjNum 1089 snap 334 seq_num 1089) 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3649: rollback oid 5 current snap is 348 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 5 to 343 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 13 current snap is 348 2024-09-13T14:31:22.314 INFO:tasks.rados.rados.0.smithi026.stdout:3650: seq_num 1149 ranges {4055040=49152} 2024-09-13T14:31:22.336 INFO:tasks.rados.rados.0.smithi026.stdout:3650: writing smithi02683976-13 from 4055040 to 4104192 tid 1 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 348 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:3651: seq_num 1150 ranges {0=65536} 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:3651: writing smithi02683976-15 from 0 to 65536 tid 1 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:3652: rollback oid 11 current snap is 348 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 11 to 337 2024-09-13T14:31:22.337 INFO:tasks.rados.rados.0.smithi026.stdout:3648: expect (ObjNum 1089 snap 334 seq_num 1089) 2024-09-13T14:31:22.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:22 smithi026 ceph-mon[99461]: pgmap v757: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 621 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-13T14:31:22.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:22 smithi026 ceph-mon[99461]: osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:31:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:22 smithi026 ceph-mon[97022]: pgmap v757: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 621 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-13T14:31:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:22 smithi026 ceph-mon[97022]: osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:31:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:22 smithi145 ceph-mon[89145]: pgmap v757: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 621 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-13T14:31:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:22 smithi145 ceph-mon[89145]: osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3649: finishing rollback tid 0 to smithi02683976-5 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3649: finishing rollback tid 1 to smithi02683976-5 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3649: finishing rollback tid 2 to smithi02683976-5 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2361 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3648: done (5 left) 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3649: done (4 left) 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3653: snap_create 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3651: finishing write tid 1 to smithi02683976-15 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3651: finishing write tid 2 to smithi02683976-15 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3652: finishing rollback tid 1 to smithi02683976-11 2024-09-13T14:31:22.512 INFO:tasks.rados.rados.0.smithi026.stdout:3652: finishing rollback tid 2 to smithi02683976-11 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1907 (ObjNum 1048 snap 320 seq_num 1048) dirty exists 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3650: finishing write tid 1 to smithi02683976-13 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3650: finishing write tid 2 to smithi02683976-13 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3650: finishing write tid 3 to smithi02683976-13 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 2046 (ObjNum 1149 snap 348 seq_num 1149) dirty exists 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3650: left oid 13 (ObjNum 1149 snap 348 seq_num 1149) 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3651: finishing write tid 3 to smithi02683976-15 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1852 (ObjNum 1150 snap 348 seq_num 1150) dirty exists 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3651: left oid 15 (ObjNum 1150 snap 348 seq_num 1150) 2024-09-13T14:31:22.513 INFO:tasks.rados.rados.0.smithi026.stdout:3647: expect (ObjNum 1099 snap 334 seq_num 1099) 2024-09-13T14:31:24.139 INFO:tasks.rados.rados.0.smithi026.stdout:3647: done (4 left) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3650: done (3 left) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3651: done (2 left) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3652: done (1 left) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3653: done (0 left) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3654: rollback oid 6 current snap is 349 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 6 to 342 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3655: read oid 16 snap -1 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3655: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:31:24.140 INFO:tasks.rados.rados.0.smithi026.stdout:3656: snap_remove snap 334 2024-09-13T14:31:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[99461]: pgmap v759: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 606 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-13T14:31:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[99461]: osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:31:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[99461]: osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:31:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[97022]: pgmap v759: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 606 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-13T14:31:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[97022]: osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:31:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:24 smithi026 ceph-mon[97022]: osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:31:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:24 smithi145 ceph-mon[89145]: pgmap v759: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 606 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.8 MiB/s wr, 12 op/s 2024-09-13T14:31:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:24 smithi145 ceph-mon[89145]: osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:31:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:24 smithi145 ceph-mon[89145]: osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:31:25.131 INFO:tasks.rados.rados.0.smithi026.stdout:3656: done (2 left) 2024-09-13T14:31:25.131 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 3 current snap is 349 2024-09-13T14:31:25.131 INFO:tasks.rados.rados.0.smithi026.stdout:3657: seq_num 1151 ranges {3923968=81920} 2024-09-13T14:31:25.153 INFO:tasks.rados.rados.0.smithi026.stdout:3657: writing smithi02683976-3 from 3923968 to 4005888 tid 1 2024-09-13T14:31:25.154 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 349 2024-09-13T14:31:25.154 INFO:tasks.rados.rados.0.smithi026.stdout:3658: seq_num 1152 ranges {3604480=98304} 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3658: writing smithi02683976-33 from 3604480 to 3702784 tid 1 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3659: rollback oid 2 current snap is 349 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 345 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3654: finishing rollback tid 0 to smithi02683976-6 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3654: finishing rollback tid 1 to smithi02683976-6 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3654: finishing rollback tid 2 to smithi02683976-6 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1911 (ObjNum 1039 snap 319 seq_num 1039) dirty exists 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3654: done (4 left) 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3660: snap_create 2024-09-13T14:31:25.175 INFO:tasks.rados.rados.0.smithi026.stdout:3655: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3657: finishing write tid 1 to smithi02683976-3 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3657: finishing write tid 2 to smithi02683976-3 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3657: finishing write tid 3 to smithi02683976-3 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 3 v 2370 (ObjNum 1151 snap 349 seq_num 1151) dirty exists 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3657: left oid 3 (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3658: finishing write tid 1 to smithi02683976-33 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3659: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:31:25.213 INFO:tasks.rados.rados.0.smithi026.stdout:3658: finishing write tid 2 to smithi02683976-33 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:3659: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:3659: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 2021 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:3658: finishing write tid 3 to smithi02683976-33 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2373 (ObjNum 1152 snap 349 seq_num 1152) dirty exists 2024-09-13T14:31:25.214 INFO:tasks.rados.rados.0.smithi026.stdout:3658: left oid 33 (ObjNum 1152 snap 349 seq_num 1152) 2024-09-13T14:31:26.135 INFO:tasks.rados.rados.0.smithi026.stdout:3655: done (4 left) 2024-09-13T14:31:26.135 INFO:tasks.rados.rados.0.smithi026.stdout:3657: done (3 left) 2024-09-13T14:31:26.135 INFO:tasks.rados.rados.0.smithi026.stdout:3658: done (2 left) 2024-09-13T14:31:26.136 INFO:tasks.rados.rados.0.smithi026.stdout:3659: done (1 left) 2024-09-13T14:31:26.136 INFO:tasks.rados.rados.0.smithi026.stdout:3660: done (0 left) 2024-09-13T14:31:26.136 INFO:tasks.rados.rados.0.smithi026.stdout:3661: read oid 5 snap -1 2024-09-13T14:31:26.136 INFO:tasks.rados.rados.0.smithi026.stdout:3661: expect (ObjNum 1006 snap 311 seq_num 1006) 2024-09-13T14:31:26.136 INFO:tasks.rados.rados.0.smithi026.stdout:3662: setattr oid 6 current snap is 350 2024-09-13T14:31:26.137 INFO:tasks.rados.rados.0.smithi026.stdout:3663: setattr oid 35 current snap is 350 2024-09-13T14:31:26.140 INFO:tasks.rados.rados.0.smithi026.stdout:3664: snap_create 2024-09-13T14:31:26.145 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1923 (ObjNum 1039 snap 319 seq_num 1039) dirty exists 2024-09-13T14:31:26.146 INFO:tasks.rados.rados.0.smithi026.stdout:3661: expect (ObjNum 1006 snap 311 seq_num 1006) 2024-09-13T14:31:26.309 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2374 (ObjNum 33 snap 0 seq_num 520231040) dirty exists 2024-09-13T14:31:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:26 smithi026 ceph-mon[97022]: pgmap v762: 121 pgs: 121 active+clean; 587 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 433 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:31:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:26 smithi026 ceph-mon[97022]: osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:31:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:26 smithi026 ceph-mon[99461]: pgmap v762: 121 pgs: 121 active+clean; 587 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 433 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:31:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:26 smithi026 ceph-mon[99461]: osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:31:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:26 smithi145 ceph-mon[89145]: pgmap v762: 121 pgs: 121 active+clean; 587 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 433 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-09-13T14:31:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:26 smithi145 ceph-mon[89145]: osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3661: done (3 left) 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3662: done (2 left) 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3663: done (1 left) 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3664: done (0 left) 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3665: rollback oid 1 current snap is 351 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 348 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3666: rollback oid 18 current snap is 351 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 333 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3667: copy_from oid 6 from oid 4 current snap is 351 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 351 2024-09-13T14:31:27.142 INFO:tasks.rados.rados.0.smithi026.stdout:3668: seq_num 1153 ranges {65536=114688} 2024-09-13T14:31:27.143 INFO:tasks.rados.rados.0.smithi026.stdout:3668: writing smithi02683976-15 from 65536 to 180224 tid 1 2024-09-13T14:31:27.143 INFO:tasks.rados.rados.0.smithi026.stdout:3669: read oid 39 snap 348 2024-09-13T14:31:27.143 INFO:tasks.rados.rados.0.smithi026.stdout:3669: expect deleted 2024-09-13T14:31:27.143 INFO:tasks.rados.rados.0.smithi026.stdout:3670: snap_create 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:3666: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:3665: finishing rollback tid 0 to smithi02683976-1 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:3666: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:3665: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:3666: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:31:27.149 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 2024 (ObjNum 1065 snap 329 seq_num 1065) dirty exists 2024-09-13T14:31:27.150 INFO:tasks.rados.rados.0.smithi026.stdout:3668: finishing write tid 1 to smithi02683976-15 2024-09-13T14:31:27.150 INFO:tasks.rados.rados.0.smithi026.stdout:3665: finishing rollback tid 2 to smithi02683976-1 2024-09-13T14:31:27.151 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 2378 (ObjNum 1089 snap 334 seq_num 1089) dirty exists 2024-09-13T14:31:27.151 INFO:tasks.rados.rados.0.smithi026.stdout:3668: finishing write tid 2 to smithi02683976-15 2024-09-13T14:31:27.156 INFO:tasks.rados.rados.0.smithi026.stdout:3668: finishing write tid 3 to smithi02683976-15 2024-09-13T14:31:27.156 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1854 (ObjNum 1153 snap 351 seq_num 1153) dirty exists 2024-09-13T14:31:27.156 INFO:tasks.rados.rados.0.smithi026.stdout:3668: left oid 15 (ObjNum 1153 snap 351 seq_num 1153) 2024-09-13T14:31:27.162 INFO:tasks.rados.rados.0.smithi026.stdout:3667: finishing copy_from racing read to smithi02683976-6 2024-09-13T14:31:27.163 INFO:tasks.rados.rados.0.smithi026.stdout:3667: finishing copy_from to smithi02683976-6 2024-09-13T14:31:27.164 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 6 v 1846 (ObjNum 1126 snap 340 seq_num 1126) dirty exists 2024-09-13T14:31:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:27 smithi026 ceph-mon[97022]: osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:31:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:27 smithi026 ceph-mon[99461]: osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:31:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:27 smithi145 ceph-mon[89145]: osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:31:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:28.142 INFO:tasks.rados.rados.0.smithi026.stdout:3665: done (5 left) 2024-09-13T14:31:28.142 INFO:tasks.rados.rados.0.smithi026.stdout:3666: done (4 left) 2024-09-13T14:31:28.142 INFO:tasks.rados.rados.0.smithi026.stdout:3667: done (3 left) 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3668: done (2 left) 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3669: done (1 left) 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3670: done (0 left) 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3671: rmattr oid 27 current snap is 352 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3671: done (0 left) 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3672: copy_from oid 33 from oid 29 current snap is 352 2024-09-13T14:31:28.143 INFO:tasks.rados.rados.0.smithi026.stdout:3673: setattr oid 37 current snap is 352 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:3674: read oid 23 snap -1 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:3674: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:3675: rollback oid 27 current snap is 352 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 285 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 352 2024-09-13T14:31:28.144 INFO:tasks.rados.rados.0.smithi026.stdout:3676: seq_num 1154 ranges {12083200=131072} 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:3676: writing smithi02683976-26 from 12083200 to 12214272 tid 1 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:3677: rollback oid 45 current snap is 352 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 45 to 285 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1487 (ObjNum 49 snap 0 seq_num 1532383341) dirty exists 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:3672: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:3675: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:3675: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:31:28.212 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2381 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:31:28.213 INFO:tasks.rados.rados.0.smithi026.stdout:3674: expect (ObjNum 1136 snap 345 seq_num 1136) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3672: finishing copy_from to smithi02683976-33 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2021 (ObjNum 1109 snap 335 seq_num 1109) dirty exists 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3677: finishing rollback tid 0 to smithi02683976-45 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3677: finishing rollback tid 1 to smithi02683976-45 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 45 v 1209 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3676: finishing write tid 1 to smithi02683976-26 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3672: done (5 left) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3673: done (4 left) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3674: done (3 left) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3675: done (2 left) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3677: done (1 left) 2024-09-13T14:31:28.345 INFO:tasks.rados.rados.0.smithi026.stdout:3678: setattr oid 50 current snap is 352 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:3676: finishing write tid 2 to smithi02683976-26 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:3676: finishing write tid 3 to smithi02683976-26 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 2031 (ObjNum 1154 snap 352 seq_num 1154) dirty exists 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:3676: left oid 26 (ObjNum 1154 snap 352 seq_num 1154) 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:3676: done (1 left) 2024-09-13T14:31:28.346 INFO:tasks.rados.rados.0.smithi026.stdout:3679: snap_create 2024-09-13T14:31:28.349 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 1927 (ObjNum 1128 snap 340 seq_num 1128) dirty exists 2024-09-13T14:31:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:28 smithi026 ceph-mon[97022]: pgmap v765: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 616 KiB/s wr, 6 op/s 2024-09-13T14:31:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:28 smithi026 ceph-mon[97022]: osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:31:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:28 smithi026 ceph-mon[99461]: pgmap v765: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 616 KiB/s wr, 6 op/s 2024-09-13T14:31:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:28 smithi026 ceph-mon[99461]: osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:31:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:28 smithi145 ceph-mon[89145]: pgmap v765: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 616 KiB/s wr, 6 op/s 2024-09-13T14:31:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:28 smithi145 ceph-mon[89145]: osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:31:29.034 INFO:tasks.rados.rados.0.smithi026.stdout:3678: done (1 left) 2024-09-13T14:31:29.034 INFO:tasks.rados.rados.0.smithi026.stdout:3679: done (0 left) 2024-09-13T14:31:29.034 INFO:tasks.rados.rados.0.smithi026.stdout:3680: read oid 20 snap -1 2024-09-13T14:31:29.034 INFO:tasks.rados.rados.0.smithi026.stdout:3680: expect deleted 2024-09-13T14:31:29.034 INFO:tasks.rados.rados.0.smithi026.stdout:3681: snap_create 2024-09-13T14:31:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:29 smithi026 ceph-mon[99461]: osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:31:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:29 smithi026 ceph-mon[99461]: osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:31:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:29 smithi026 ceph-mon[97022]: osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:31:29.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:29 smithi026 ceph-mon[97022]: osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:31:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:29 smithi145 ceph-mon[89145]: osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:31:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:29 smithi145 ceph-mon[89145]: osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:31:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:3680: done (1 left) 2024-09-13T14:31:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:3681: done (0 left) 2024-09-13T14:31:30.035 INFO:tasks.rados.rados.0.smithi026.stdout:3682: snap_create 2024-09-13T14:31:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:30 smithi026 ceph-mon[97022]: pgmap v768: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-13T14:31:30.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:30 smithi026 ceph-mon[97022]: osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:31:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:30 smithi026 ceph-mon[99461]: pgmap v768: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-13T14:31:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:30 smithi026 ceph-mon[99461]: osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:31:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:30 smithi145 ceph-mon[89145]: pgmap v768: 121 pgs: 121 active+clean; 520 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-13T14:31:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:30 smithi145 ceph-mon[89145]: osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:31:30.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:31.036 INFO:tasks.rados.rados.0.smithi026.stdout:3682: done (0 left) 2024-09-13T14:31:31.036 INFO:tasks.rados.rados.0.smithi026.stdout:3683: rmattr oid 8 current snap is 355 2024-09-13T14:31:31.036 INFO:tasks.rados.rados.0.smithi026.stdout:3684: copy_from oid 30 from oid 7 current snap is 355 2024-09-13T14:31:31.037 INFO:tasks.rados.rados.0.smithi026.stdout:3685: setattr oid 47 current snap is 355 2024-09-13T14:31:31.038 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 355 2024-09-13T14:31:31.038 INFO:tasks.rados.rados.0.smithi026.stdout:3686: seq_num 1155 ranges {0=147456} 2024-09-13T14:31:31.039 INFO:tasks.rados.rados.0.smithi026.stdout:3686: writing smithi02683976-20 from 0 to 147456 tid 1 2024-09-13T14:31:31.039 INFO:tasks.rados.rados.0.smithi026.stdout:3687: snap_remove snap 338 2024-09-13T14:31:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:31] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.51.0" 2024-09-13T14:31:32.041 INFO:tasks.rados.rados.0.smithi026.stdout:3687: done (4 left) 2024-09-13T14:31:32.041 INFO:tasks.rados.rados.0.smithi026.stdout:3688: read oid 16 snap 343 2024-09-13T14:31:32.042 INFO:tasks.rados.rados.0.smithi026.stdout:3688: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:31:32.042 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 11 current snap is 355 2024-09-13T14:31:32.042 INFO:tasks.rados.rados.0.smithi026.stdout:3689: seq_num 1156 ranges {761856=163840} 2024-09-13T14:31:32.047 INFO:tasks.rados.rados.0.smithi026.stdout:3689: writing smithi02683976-11 from 761856 to 925696 tid 1 2024-09-13T14:31:32.048 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 0 (ObjNum 33 snap 0 seq_num 518851856) dirty dne 2024-09-13T14:31:32.048 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 1483 (ObjNum 65 snap 0 seq_num 509658064) dirty exists 2024-09-13T14:31:32.048 INFO:tasks.rados.rados.0.smithi026.stdout:3686: finishing write tid 1 to smithi02683976-20 2024-09-13T14:31:32.048 INFO:tasks.rados.rados.0.smithi026.stdout:3686: finishing write tid 2 to smithi02683976-20 2024-09-13T14:31:32.048 INFO:tasks.rados.rados.0.smithi026.stdout:3686: finishing write tid 3 to smithi02683976-20 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2048 (ObjNum 1155 snap 355 seq_num 1155) dirty exists 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3686: left oid 20 (ObjNum 1155 snap 355 seq_num 1155) 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3684: finishing copy_from racing read to smithi02683976-30 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3684: finishing copy_from to smithi02683976-30 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1613 (ObjNum 1138 snap 345 seq_num 1138) dirty exists 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3683: done (5 left) 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3684: done (4 left) 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3685: done (3 left) 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3686: done (2 left) 2024-09-13T14:31:32.049 INFO:tasks.rados.rados.0.smithi026.stdout:3690: snap_remove snap 352 2024-09-13T14:31:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:32 smithi026 ceph-mon[97022]: pgmap v771: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 28 op/s 2024-09-13T14:31:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:32 smithi026 ceph-mon[97022]: osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:31:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:32 smithi026 ceph-mon[99461]: pgmap v771: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 28 op/s 2024-09-13T14:31:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:32 smithi026 ceph-mon[99461]: osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:31:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:32 smithi145 ceph-mon[89145]: pgmap v771: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 28 op/s 2024-09-13T14:31:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:32 smithi145 ceph-mon[89145]: osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:31:33.047 INFO:tasks.rados.rados.0.smithi026.stdout:3690: done (2 left) 2024-09-13T14:31:33.047 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 355 2024-09-13T14:31:33.047 INFO:tasks.rados.rados.0.smithi026.stdout:3689: finishing write tid 1 to smithi02683976-11 2024-09-13T14:31:33.047 INFO:tasks.rados.rados.0.smithi026.stdout:3689: finishing write tid 2 to smithi02683976-11 2024-09-13T14:31:33.048 INFO:tasks.rados.rados.0.smithi026.stdout:3689: finishing write tid 3 to smithi02683976-11 2024-09-13T14:31:33.048 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 11 v 1934 (ObjNum 1156 snap 355 seq_num 1156) dirty exists 2024-09-13T14:31:33.048 INFO:tasks.rados.rados.0.smithi026.stdout:3689: left oid 11 (ObjNum 1156 snap 355 seq_num 1156) 2024-09-13T14:31:33.048 INFO:tasks.rados.rados.0.smithi026.stdout:3688: expect (ObjNum 798 snap 245 seq_num 798) 2024-09-13T14:31:33.083 INFO:tasks.rados.rados.0.smithi026.stdout:3691: seq_num 1157 ranges {8200192=180224} 2024-09-13T14:31:33.129 INFO:tasks.rados.rados.0.smithi026.stdout:3691: writing smithi02683976-25 from 8200192 to 8380416 tid 1 2024-09-13T14:31:33.129 INFO:tasks.rados.rados.0.smithi026.stdout:3688: done (2 left) 2024-09-13T14:31:33.129 INFO:tasks.rados.rados.0.smithi026.stdout:3689: done (1 left) 2024-09-13T14:31:33.129 INFO:tasks.rados.rados.0.smithi026.stdout:3692: snap_create 2024-09-13T14:31:33.136 INFO:tasks.rados.rados.0.smithi026.stdout:3691: finishing write tid 1 to smithi02683976-25 2024-09-13T14:31:33.136 INFO:tasks.rados.rados.0.smithi026.stdout:3691: finishing write tid 2 to smithi02683976-25 2024-09-13T14:31:33.137 INFO:tasks.rados.rados.0.smithi026.stdout:3691: finishing write tid 3 to smithi02683976-25 2024-09-13T14:31:33.137 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2023 (ObjNum 1157 snap 355 seq_num 1157) dirty exists 2024-09-13T14:31:33.137 INFO:tasks.rados.rados.0.smithi026.stdout:3691: left oid 25 (ObjNum 1157 snap 355 seq_num 1157) 2024-09-13T14:31:33.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:33 smithi026 ceph-mon[97022]: osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:31:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:33 smithi026 ceph-mon[99461]: osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:31:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:33 smithi145 ceph-mon[89145]: osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:31:34.035 INFO:tasks.rados.rados.0.smithi026.stdout:3691: done (1 left) 2024-09-13T14:31:34.035 INFO:tasks.rados.rados.0.smithi026.stdout:3692: done (0 left) 2024-09-13T14:31:34.035 INFO:tasks.rados.rados.0.smithi026.stdout:3693: snap_create 2024-09-13T14:31:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[97022]: pgmap v774: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-09-13T14:31:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[97022]: osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:31:34.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[97022]: osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:31:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[99461]: pgmap v774: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-09-13T14:31:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[99461]: osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:31:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:34 smithi026 ceph-mon[99461]: osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:31:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:34 smithi145 ceph-mon[89145]: pgmap v774: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-09-13T14:31:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:34 smithi145 ceph-mon[89145]: osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:31:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:34 smithi145 ceph-mon[89145]: osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3693: done (0 left) 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3694: rollback oid 29 current snap is 357 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 29 to 348 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3695: rmattr oid 15 current snap is 357 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3695: done (1 left) 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3696: rmattr oid 40 current snap is 357 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3696: done (1 left) 2024-09-13T14:31:35.037 INFO:tasks.rados.rados.0.smithi026.stdout:3697: snap_remove snap 335 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:3697: done (1 left) 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:3698: delete oid 12 current snap is 357 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:3694: finishing rollback tid 0 to smithi02683976-29 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:3694: finishing rollback tid 1 to smithi02683976-29 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:3694: finishing rollback tid 2 to smithi02683976-29 2024-09-13T14:31:36.040 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 2051 (ObjNum 1109 snap 335 seq_num 1109) dirty exists 2024-09-13T14:31:36.052 INFO:tasks.rados.rados.0.smithi026.stdout:3694: done (1 left) 2024-09-13T14:31:36.052 INFO:tasks.rados.rados.0.smithi026.stdout:3698: done (0 left) 2024-09-13T14:31:36.052 INFO:tasks.rados.rados.0.smithi026.stdout:3699: snap_create 2024-09-13T14:31:36.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:36 smithi026 ceph-mon[99461]: pgmap v777: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 76 KiB/s wr, 1 op/s 2024-09-13T14:31:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:36 smithi026 ceph-mon[99461]: osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:31:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:36 smithi026 ceph-mon[97022]: pgmap v777: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 76 KiB/s wr, 1 op/s 2024-09-13T14:31:36.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:36 smithi026 ceph-mon[97022]: osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:31:36.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:36 smithi145 ceph-mon[89145]: pgmap v777: 121 pgs: 121 active+clean; 525 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 76 KiB/s wr, 1 op/s 2024-09-13T14:31:36.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:36 smithi145 ceph-mon[89145]: osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:31:37.051 INFO:tasks.rados.rados.0.smithi026.stdout:3699: done (0 left) 2024-09-13T14:31:37.051 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 19 current snap is 358 2024-09-13T14:31:37.052 INFO:tasks.rados.rados.0.smithi026.stdout:3700: seq_num 1159 ranges {6160384=212992} 2024-09-13T14:31:37.086 INFO:tasks.rados.rados.0.smithi026.stdout:3700: writing smithi02683976-19 from 6160384 to 6373376 tid 1 2024-09-13T14:31:37.086 INFO:tasks.rados.rados.0.smithi026.stdout:3701: copy_from oid 20 from oid 49 current snap is 358 2024-09-13T14:31:37.086 INFO:tasks.rados.rados.0.smithi026.stdout:3702: snap_remove snap 285 2024-09-13T14:31:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:37 smithi026 ceph-mon[97022]: osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:31:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:37 smithi026 ceph-mon[99461]: osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:31:37.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:37 smithi145 ceph-mon[89145]: osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3702: done (2 left) 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3703: rollback oid 39 current snap is 358 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 39 to 344 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3700: finishing write tid 1 to smithi02683976-19 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3700: finishing write tid 2 to smithi02683976-19 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3701: finishing copy_from to smithi02683976-20 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3701: got expected ENOENT (src dne) 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3701: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:31:38.054 INFO:tasks.rados.rados.0.smithi026.stdout:3700: finishing write tid 3 to smithi02683976-19 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2053 (ObjNum 1159 snap 358 seq_num 1159) dirty exists 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:3700: left oid 19 (ObjNum 1159 snap 358 seq_num 1159) 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:3700: done (2 left) 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:3701: done (1 left) 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:3704: copy_from oid 27 from oid 46 current snap is 358 2024-09-13T14:31:38.055 INFO:tasks.rados.rados.0.smithi026.stdout:3705: delete oid 4 current snap is 358 2024-09-13T14:31:38.066 INFO:tasks.rados.rados.0.smithi026.stdout:3705: done (2 left) 2024-09-13T14:31:38.066 INFO:tasks.rados.rados.0.smithi026.stdout:3706: rollback oid 38 current snap is 358 2024-09-13T14:31:38.066 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 350 2024-09-13T14:31:38.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 23 current snap is 358 2024-09-13T14:31:38.066 INFO:tasks.rados.rados.0.smithi026.stdout:3707: seq_num 1161 ranges {3973120=245760} 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3707: writing smithi02683976-23 from 3973120 to 4218880 tid 1 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 358 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3703: finishing rollback tid 1 to smithi02683976-39 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 2007 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3706: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3706: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3706: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 2027 (ObjNum 977 snap 306 seq_num 977) dirty exists 2024-09-13T14:31:38.091 INFO:tasks.rados.rados.0.smithi026.stdout:3708: seq_num 1162 ranges {0=270336} 2024-09-13T14:31:38.093 INFO:tasks.rados.rados.0.smithi026.stdout:3708: writing smithi02683976-43 from 0 to 270336 tid 1 2024-09-13T14:31:38.093 INFO:tasks.rados.rados.0.smithi026.stdout:3703: done (4 left) 2024-09-13T14:31:38.093 INFO:tasks.rados.rados.0.smithi026.stdout:3706: done (3 left) 2024-09-13T14:31:38.093 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 20 current snap is 358 2024-09-13T14:31:38.093 INFO:tasks.rados.rados.0.smithi026.stdout:3709: seq_num 1163 ranges {147456=286720} 2024-09-13T14:31:38.096 INFO:tasks.rados.rados.0.smithi026.stdout:3709: writing smithi02683976-20 from 147456 to 434176 tid 1 2024-09-13T14:31:38.096 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 358 2024-09-13T14:31:38.096 INFO:tasks.rados.rados.0.smithi026.stdout:3710: seq_num 1164 ranges {1736704=303104} 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3710: writing smithi02683976-44 from 1736704 to 2039808 tid 1 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3711: delete oid 28 current snap is 358 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3707: finishing write tid 1 to smithi02683976-23 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3707: finishing write tid 2 to smithi02683976-23 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3708: finishing write tid 1 to smithi02683976-43 2024-09-13T14:31:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3708: finishing write tid 2 to smithi02683976-43 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3709: finishing write tid 1 to smithi02683976-20 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3704: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3708: finishing write tid 3 to smithi02683976-43 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2055 (ObjNum 1162 snap 358 seq_num 1162) dirty exists 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3708: left oid 43 (ObjNum 1162 snap 358 seq_num 1162) 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3707: finishing write tid 3 to smithi02683976-23 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 2029 (ObjNum 1161 snap 358 seq_num 1161) dirty exists 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3707: left oid 23 (ObjNum 1161 snap 358 seq_num 1161) 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3709: finishing write tid 2 to smithi02683976-20 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3704: finishing copy_from to smithi02683976-27 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 1656 (ObjNum 1139 snap 347 seq_num 1139) dirty exists 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3709: finishing write tid 3 to smithi02683976-20 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2057 (ObjNum 1163 snap 358 seq_num 1163) dirty exists 2024-09-13T14:31:38.110 INFO:tasks.rados.rados.0.smithi026.stdout:3709: left oid 20 (ObjNum 1163 snap 358 seq_num 1163) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3704: done (5 left) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3707: done (4 left) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3708: done (3 left) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3709: done (2 left) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3711: done (1 left) 2024-09-13T14:31:38.111 INFO:tasks.rados.rados.0.smithi026.stdout:3712: delete oid 1 current snap is 358 2024-09-13T14:31:38.112 INFO:tasks.rados.rados.0.smithi026.stdout:3710: finishing write tid 1 to smithi02683976-44 2024-09-13T14:31:38.112 INFO:tasks.rados.rados.0.smithi026.stdout:3710: finishing write tid 2 to smithi02683976-44 2024-09-13T14:31:38.112 INFO:tasks.rados.rados.0.smithi026.stdout:3710: finishing write tid 3 to smithi02683976-44 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1658 (ObjNum 1164 snap 358 seq_num 1164) dirty exists 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:3710: left oid 44 (ObjNum 1164 snap 358 seq_num 1164) 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:3710: done (1 left) 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:3712: done (0 left) 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 358 2024-09-13T14:31:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:3713: seq_num 1167 ranges {0=352256} 2024-09-13T14:31:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:3713: writing smithi02683976-37 from 0 to 352256 tid 1 2024-09-13T14:31:38.115 INFO:tasks.rados.rados.0.smithi026.stdout:3714: snap_create 2024-09-13T14:31:38.118 INFO:tasks.rados.rados.0.smithi026.stdout:3713: finishing write tid 1 to smithi02683976-37 2024-09-13T14:31:38.118 INFO:tasks.rados.rados.0.smithi026.stdout:3713: finishing write tid 2 to smithi02683976-37 2024-09-13T14:31:38.119 INFO:tasks.rados.rados.0.smithi026.stdout:3713: finishing write tid 3 to smithi02683976-37 2024-09-13T14:31:38.119 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1489 (ObjNum 1167 snap 358 seq_num 1167) dirty exists 2024-09-13T14:31:38.119 INFO:tasks.rados.rados.0.smithi026.stdout:3713: left oid 37 (ObjNum 1167 snap 358 seq_num 1167) 2024-09-13T14:31:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:38 smithi026 ceph-mon[97022]: pgmap v780: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:38.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:38 smithi026 ceph-mon[97022]: osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:31:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:38 smithi026 ceph-mon[99461]: pgmap v780: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:38.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:38 smithi026 ceph-mon[99461]: osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:31:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:38 smithi145 ceph-mon[89145]: pgmap v780: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:38 smithi145 ceph-mon[89145]: osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3713: done (1 left) 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3714: done (0 left) 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3715: read oid 25 snap -1 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3715: expect (ObjNum 1157 snap 355 seq_num 1157) 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3716: rollback oid 21 current snap is 359 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 327 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3717: read oid 35 snap 353 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3717: expect (ObjNum 520156944 snap 21981 seq_num 519073984) 2024-09-13T14:31:39.038 INFO:tasks.rados.rados.0.smithi026.stdout:3718: read oid 18 snap -1 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:3718: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:3719: rollback oid 2 current snap is 359 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 2 to 337 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:3720: copy_from oid 39 from oid 34 current snap is 359 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 359 2024-09-13T14:31:39.039 INFO:tasks.rados.rados.0.smithi026.stdout:3721: seq_num 1168 ranges {0=368640} 2024-09-13T14:31:39.040 INFO:tasks.rados.rados.0.smithi026.stdout:3721: writing smithi02683976-14 from 0 to 368640 tid 1 2024-09-13T14:31:39.040 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 359 2024-09-13T14:31:39.040 INFO:tasks.rados.rados.0.smithi026.stdout:3722: seq_num 1169 ranges {0=385024} 2024-09-13T14:31:39.043 INFO:tasks.rados.rados.0.smithi026.stdout:3722: writing smithi02683976-28 from 0 to 385024 tid 1 2024-09-13T14:31:39.043 INFO:tasks.rados.rados.0.smithi026.stdout:3723: rmattr oid 19 current snap is 359 2024-09-13T14:31:39.043 INFO:tasks.rados.rados.0.smithi026.stdout:3723: done (8 left) 2024-09-13T14:31:39.043 INFO:tasks.rados.rados.0.smithi026.stdout:3724: snap_remove snap 301 2024-09-13T14:31:39.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:39 smithi026 ceph-mon[99461]: osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:31:39.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:39 smithi026 ceph-mon[99461]: osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:31:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:39 smithi026 ceph-mon[97022]: osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:31:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:39 smithi026 ceph-mon[97022]: osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:31:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:39 smithi145 ceph-mon[89145]: osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:31:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:39 smithi145 ceph-mon[89145]: osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:31:40.039 INFO:tasks.rados.rados.0.smithi026.stdout:3724: done (8 left) 2024-09-13T14:31:40.039 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 359 2024-09-13T14:31:40.040 INFO:tasks.rados.rados.0.smithi026.stdout:3725: seq_num 1170 ranges {2154496=401408} 2024-09-13T14:31:40.055 INFO:tasks.rados.rados.0.smithi026.stdout:3725: writing smithi02683976-17 from 2154496 to 2555904 tid 1 2024-09-13T14:31:40.055 INFO:tasks.rados.rados.0.smithi026.stdout:3726: snap_remove snap 357 2024-09-13T14:31:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:40 smithi026 ceph-mon[97022]: pgmap v783: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:40 smithi026 ceph-mon[97022]: osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:31:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:40 smithi026 ceph-mon[99461]: pgmap v783: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:40 smithi026 ceph-mon[99461]: osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:31:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:40 smithi145 ceph-mon[89145]: pgmap v783: 121 pgs: 121 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1008 KiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-13T14:31:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:40 smithi145 ceph-mon[89145]: osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:31:40.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:41.041 INFO:tasks.rados.rados.0.smithi026.stdout:3726: done (9 left) 2024-09-13T14:31:41.041 INFO:tasks.rados.rados.0.smithi026.stdout:3727: delete oid 16 current snap is 359 2024-09-13T14:31:41.041 INFO:tasks.rados.rados.0.smithi026.stdout:3717: expect (ObjNum 520156944 snap 21981 seq_num 519073984) 2024-09-13T14:31:41.046 INFO:tasks.rados.rados.0.smithi026.stdout:3718: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3722: finishing write tid 1 to smithi02683976-28 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3721: finishing write tid 1 to smithi02683976-14 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3722: finishing write tid 2 to smithi02683976-28 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3721: finishing write tid 2 to smithi02683976-14 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3716: finishing rollback tid 0 to smithi02683976-21 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3716: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3716: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 1958 (ObjNum 65 snap 0 seq_num 519600432) dirty exists 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3722: finishing write tid 3 to smithi02683976-28 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 2037 (ObjNum 1169 snap 359 seq_num 1169) dirty exists 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3722: left oid 28 (ObjNum 1169 snap 359 seq_num 1169) 2024-09-13T14:31:41.081 INFO:tasks.rados.rados.0.smithi026.stdout:3721: finishing write tid 3 to smithi02683976-14 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 2031 (ObjNum 1168 snap 359 seq_num 1168) dirty exists 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3721: left oid 14 (ObjNum 1168 snap 359 seq_num 1168) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3716: done (9 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3717: done (8 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3718: done (7 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3721: done (6 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3722: done (5 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3727: done (4 left) 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3728: rollback oid 18 current snap is 359 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 18 to 354 2024-09-13T14:31:41.082 INFO:tasks.rados.rados.0.smithi026.stdout:3715: expect (ObjNum 1157 snap 355 seq_num 1157) 2024-09-13T14:31:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:41] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:31:41.749 INFO:tasks.rados.rados.0.smithi026.stdout:3719: finishing rollback tid 0 to smithi02683976-2 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3719: finishing rollback tid 1 to smithi02683976-2 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3715: done (4 left) 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3729: read oid 25 snap -1 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3729: expect (ObjNum 1157 snap 355 seq_num 1157) 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3730: delete oid 23 current snap is 359 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3719: finishing rollback tid 2 to smithi02683976-2 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 2 v 2026 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3720: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3720: finishing copy_from to smithi02683976-39 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1930 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3725: finishing write tid 1 to smithi02683976-17 2024-09-13T14:31:41.750 INFO:tasks.rados.rados.0.smithi026.stdout:3725: finishing write tid 2 to smithi02683976-17 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:3725: finishing write tid 3 to smithi02683976-17 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1945 (ObjNum 1170 snap 359 seq_num 1170) dirty exists 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:3725: left oid 17 (ObjNum 1170 snap 359 seq_num 1170) 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:3728: finishing rollback tid 0 to smithi02683976-18 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:3728: finishing rollback tid 1 to smithi02683976-18 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:3728: finishing rollback tid 2 to smithi02683976-18 2024-09-13T14:31:41.751 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 2034 (ObjNum 1065 snap 329 seq_num 1065) dirty exists 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3719: done (5 left) 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3720: done (4 left) 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3725: done (3 left) 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3728: done (2 left) 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3730: done (1 left) 2024-09-13T14:31:41.752 INFO:tasks.rados.rados.0.smithi026.stdout:3731: snap_remove snap 343 2024-09-13T14:31:42.070 INFO:tasks.rados.rados.0.smithi026.stdout:3731: done (1 left) 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:3732: rollback oid 27 current snap is 359 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 27 to 333 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:3733: rollback oid 36 current snap is 359 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 36 to 354 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:3734: rollback oid 48 current snap is 359 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 48 to 350 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:3735: delete oid 24 current snap is 359 2024-09-13T14:31:42.071 INFO:tasks.rados.rados.0.smithi026.stdout:3729: expect (ObjNum 1157 snap 355 seq_num 1157) 2024-09-13T14:31:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[97022]: pgmap v786: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 416 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-13T14:31:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[97022]: osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:31:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[99461]: pgmap v786: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 416 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-13T14:31:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[99461]: osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:31:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:42 smithi145 ceph-mon[89145]: pgmap v786: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 416 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-13T14:31:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:42 smithi145 ceph-mon[89145]: osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:31:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.745 INFO:tasks.rados.rados.0.smithi026.stdout:3734: finishing rollback tid 0 to smithi02683976-48 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3734: finishing rollback tid 1 to smithi02683976-48 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3734: finishing rollback tid 2 to smithi02683976-48 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 1486 (ObjNum 1050 snap 320 seq_num 1050) dirty exists 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3729: done (4 left) 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3734: done (3 left) 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3735: done (2 left) 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3736: delete oid 37 current snap is 359 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3732: finishing rollback tid 0 to smithi02683976-27 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3732: finishing rollback tid 1 to smithi02683976-27 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3732: finishing rollback tid 2 to smithi02683976-27 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2436 (ObjNum 1056 snap 323 seq_num 1056) dirty exists 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3733: finishing rollback tid 0 to smithi02683976-36 2024-09-13T14:31:42.746 INFO:tasks.rados.rados.0.smithi026.stdout:3733: finishing rollback tid 1 to smithi02683976-36 2024-09-13T14:31:42.747 INFO:tasks.rados.rados.0.smithi026.stdout:3733: finishing rollback tid 2 to smithi02683976-36 2024-09-13T14:31:42.747 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2440 (ObjNum 1098 snap 334 seq_num 1098) dirty exists 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3732: done (2 left) 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3733: done (1 left) 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3736: done (0 left) 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3737: rmattr oid 44 current snap is 359 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3737: done (0 left) 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3738: read oid 7 snap -1 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3738: expect (ObjNum 1138 snap 345 seq_num 1138) 2024-09-13T14:31:42.748 INFO:tasks.rados.rados.0.smithi026.stdout:3739: snap_create 2024-09-13T14:31:42.758 INFO:tasks.rados.rados.0.smithi026.stdout:3738: expect (ObjNum 1138 snap 345 seq_num 1138) 2024-09-13T14:31:43.073 INFO:tasks.rados.rados.0.smithi026.stdout:3738: done (1 left) 2024-09-13T14:31:43.073 INFO:tasks.rados.rados.0.smithi026.stdout:3739: done (0 left) 2024-09-13T14:31:43.073 INFO:tasks.rados.rados.0.smithi026.stdout:3740: delete oid 10 current snap is 360 2024-09-13T14:31:43.075 INFO:tasks.rados.rados.0.smithi026.stdout:3740: done (0 left) 2024-09-13T14:31:43.076 INFO:tasks.rados.rados.0.smithi026.stdout:3741: copy_from oid 32 from oid 4 current snap is 360 2024-09-13T14:31:43.076 INFO:tasks.rados.rados.0.smithi026.stdout:3742: read oid 44 snap -1 2024-09-13T14:31:43.076 INFO:tasks.rados.rados.0.smithi026.stdout:3742: expect (ObjNum 1164 snap 358 seq_num 1164) 2024-09-13T14:31:43.076 INFO:tasks.rados.rados.0.smithi026.stdout:3743: copy_from oid 20 from oid 2 current snap is 360 2024-09-13T14:31:43.076 INFO:tasks.rados.rados.0.smithi026.stdout:3744: snap_remove snap 349 2024-09-13T14:31:43.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:43 smithi026 ceph-mon[99461]: osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:31:43.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:43 smithi026 ceph-mon[97022]: osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:31:43.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:43 smithi145 ceph-mon[89145]: osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:31:44.039 INFO:tasks.rados.rados.0.smithi026.stdout:3744: done (3 left) 2024-09-13T14:31:44.039 INFO:tasks.rados.rados.0.smithi026.stdout:3745: read oid 37 snap -1 2024-09-13T14:31:44.039 INFO:tasks.rados.rados.0.smithi026.stdout:3745: expect deleted 2024-09-13T14:31:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:3746: rollback oid 7 current snap is 360 2024-09-13T14:31:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 7 to 337 2024-09-13T14:31:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:3741: finishing copy_from to smithi02683976-32 2024-09-13T14:31:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:3741: got expected ENOENT (src dne) 2024-09-13T14:31:44.040 INFO:tasks.rados.rados.0.smithi026.stdout:3747: snap_remove snap 348 2024-09-13T14:31:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[97022]: pgmap v789: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 390 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.5 MiB/s wr, 15 op/s 2024-09-13T14:31:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[97022]: osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:31:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[97022]: osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:31:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[99461]: pgmap v789: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 390 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.5 MiB/s wr, 15 op/s 2024-09-13T14:31:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[99461]: osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:31:44.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:44 smithi026 ceph-mon[99461]: osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:31:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:44 smithi145 ceph-mon[89145]: pgmap v789: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 390 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.5 MiB/s wr, 15 op/s 2024-09-13T14:31:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:44 smithi145 ceph-mon[89145]: osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:31:44.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:44 smithi145 ceph-mon[89145]: osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3747: done (5 left) 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3748: read oid 31 snap 337 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3748: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3749: copy_from oid 18 from oid 12 current snap is 360 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3750: read oid 15 snap -1 2024-09-13T14:31:45.045 INFO:tasks.rados.rados.0.smithi026.stdout:3750: expect (ObjNum 1153 snap 351 seq_num 1153) 2024-09-13T14:31:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:3751: read oid 1 snap 341 2024-09-13T14:31:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:3751: expect (ObjNum 1089 snap 334 seq_num 1089) 2024-09-13T14:31:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 360 2024-09-13T14:31:45.046 INFO:tasks.rados.rados.0.smithi026.stdout:3752: seq_num 1176 ranges {3301376=499712} 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3752: writing smithi02683976-33 from 3301376 to 3801088 tid 1 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3753: rollback oid 23 current snap is 360 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 23 to 327 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3741: finishing copy_from racing read to smithi02683976-32 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3743: finishing copy_from racing read to smithi02683976-20 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3743: finishing copy_from to smithi02683976-20 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2026 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:31:45.065 INFO:tasks.rados.rados.0.smithi026.stdout:3742: expect (ObjNum 1164 snap 358 seq_num 1164) 2024-09-13T14:31:45.157 INFO:tasks.rados.rados.0.smithi026.stdout:3746: finishing rollback tid 0 to smithi02683976-7 2024-09-13T14:31:45.157 INFO:tasks.rados.rados.0.smithi026.stdout:3746: finishing rollback tid 1 to smithi02683976-7 2024-09-13T14:31:45.157 INFO:tasks.rados.rados.0.smithi026.stdout:3746: finishing rollback tid 2 to smithi02683976-7 2024-09-13T14:31:45.157 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1616 (ObjNum 1051 snap 323 seq_num 1051) dirty exists 2024-09-13T14:31:45.157 INFO:tasks.rados.rados.0.smithi026.stdout:3750: expect (ObjNum 1153 snap 351 seq_num 1153) 2024-09-13T14:31:45.160 INFO:tasks.rados.rados.0.smithi026.stdout:3749: finishing copy_from to smithi02683976-18 2024-09-13T14:31:45.161 INFO:tasks.rados.rados.0.smithi026.stdout:3749: got expected ENOENT (src dne) 2024-09-13T14:31:45.161 INFO:tasks.rados.rados.0.smithi026.stdout:3749: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:31:45.161 INFO:tasks.rados.rados.0.smithi026.stdout:3748: expect (ObjNum 963 snap 302 seq_num 963) 2024-09-13T14:31:45.206 INFO:tasks.rados.rados.0.smithi026.stdout:3751: expect (ObjNum 1089 snap 334 seq_num 1089) 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3753: finishing rollback tid 1 to smithi02683976-23 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3753: finishing rollback tid 2 to smithi02683976-23 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 2037 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3752: finishing write tid 1 to smithi02683976-33 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3752: finishing write tid 2 to smithi02683976-33 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3752: finishing write tid 3 to smithi02683976-33 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2465 (ObjNum 1176 snap 360 seq_num 1176) dirty exists 2024-09-13T14:31:45.379 INFO:tasks.rados.rados.0.smithi026.stdout:3752: left oid 33 (ObjNum 1176 snap 360 seq_num 1176) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3741: done (10 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3742: done (9 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3743: done (8 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3745: done (7 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3746: done (6 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3748: done (5 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3749: done (4 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3750: done (3 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3751: done (2 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3752: done (1 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3753: done (0 left) 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3754: read oid 40 snap -1 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3754: expect deleted 2024-09-13T14:31:45.380 INFO:tasks.rados.rados.0.smithi026.stdout:3755: snap_create 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:3754: done (1 left) 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:3755: done (0 left) 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:3756: read oid 49 snap 356 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:3756: expect deleted 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 361 2024-09-13T14:31:46.080 INFO:tasks.rados.rados.0.smithi026.stdout:3757: seq_num 1177 ranges {12214272=516096} 2024-09-13T14:31:46.149 INFO:tasks.rados.rados.0.smithi026.stdout:3757: writing smithi02683976-26 from 12214272 to 12730368 tid 1 2024-09-13T14:31:46.149 INFO:tasks.rados.rados.0.smithi026.stdout:3758: rollback oid 34 current snap is 361 2024-09-13T14:31:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 34 to 346 2024-09-13T14:31:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3756: done (2 left) 2024-09-13T14:31:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3759: delete oid 8 current snap is 361 2024-09-13T14:31:46.154 INFO:tasks.rados.rados.0.smithi026.stdout:3759: done (2 left) 2024-09-13T14:31:46.154 INFO:tasks.rados.rados.0.smithi026.stdout:3760: read oid 45 snap -1 2024-09-13T14:31:46.154 INFO:tasks.rados.rados.0.smithi026.stdout:3760: expect deleted 2024-09-13T14:31:46.154 INFO:tasks.rados.rados.0.smithi026.stdout:3761: copy_from oid 47 from oid 18 current snap is 361 2024-09-13T14:31:46.154 INFO:tasks.rados.rados.0.smithi026.stdout:3762: snap_remove snap 337 2024-09-13T14:31:46.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:46 smithi026 ceph-mon[97022]: pgmap v792: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 382 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-09-13T14:31:46.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:46 smithi026 ceph-mon[97022]: osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:31:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:46 smithi026 ceph-mon[99461]: pgmap v792: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 382 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-09-13T14:31:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:46 smithi026 ceph-mon[99461]: osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:31:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:46 smithi145 ceph-mon[89145]: pgmap v792: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 382 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-09-13T14:31:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:46 smithi145 ceph-mon[89145]: osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:31:47.087 INFO:tasks.rados.rados.0.smithi026.stdout:3762: done (4 left) 2024-09-13T14:31:47.087 INFO:tasks.rados.rados.0.smithi026.stdout:3763: snap_remove snap 355 2024-09-13T14:31:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:47 smithi026 ceph-mon[97022]: osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:31:47.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:47 smithi026 ceph-mon[99461]: osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:31:47.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:47 smithi145 ceph-mon[89145]: osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:31:48.091 INFO:tasks.rados.rados.0.smithi026.stdout:3763: done (4 left) 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3764: rollback oid 24 current snap is 361 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 24 to 360 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3758: finishing rollback tid 0 to smithi02683976-34 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3758: finishing rollback tid 1 to smithi02683976-34 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3758: finishing rollback tid 2 to smithi02683976-34 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1948 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3757: finishing write tid 1 to smithi02683976-26 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3757: finishing write tid 2 to smithi02683976-26 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3761: finishing copy_from racing read to smithi02683976-47 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3757: finishing write tid 3 to smithi02683976-26 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 2034 (ObjNum 1177 snap 361 seq_num 1177) dirty exists 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3757: left oid 26 (ObjNum 1177 snap 361 seq_num 1177) 2024-09-13T14:31:48.092 INFO:tasks.rados.rados.0.smithi026.stdout:3761: finishing copy_from to smithi02683976-47 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 2034 (ObjNum 1065 snap 329 seq_num 1065) dirty exists 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:3757: done (4 left) 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:3758: done (3 left) 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:3760: done (2 left) 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:3761: done (1 left) 2024-09-13T14:31:48.093 INFO:tasks.rados.rados.0.smithi026.stdout:3765: delete oid 21 current snap is 361 2024-09-13T14:31:48.100 INFO:tasks.rados.rados.0.smithi026.stdout:3764: finishing rollback tid 1 to smithi02683976-24 2024-09-13T14:31:48.100 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 2027 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-13T14:31:48.103 INFO:tasks.rados.rados.0.smithi026.stdout:3764: done (1 left) 2024-09-13T14:31:48.103 INFO:tasks.rados.rados.0.smithi026.stdout:3765: done (0 left) 2024-09-13T14:31:48.104 INFO:tasks.rados.rados.0.smithi026.stdout:3766: snap_create 2024-09-13T14:31:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:48 smithi026 ceph-mon[97022]: pgmap v795: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 108 KiB/s wr, 6 op/s 2024-09-13T14:31:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:48 smithi026 ceph-mon[97022]: osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:31:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:48 smithi026 ceph-mon[99461]: pgmap v795: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 108 KiB/s wr, 6 op/s 2024-09-13T14:31:48.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:48 smithi026 ceph-mon[99461]: osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:31:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:48 smithi145 ceph-mon[89145]: pgmap v795: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 108 KiB/s wr, 6 op/s 2024-09-13T14:31:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:48 smithi145 ceph-mon[89145]: osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:31:49.041 INFO:tasks.rados.rados.0.smithi026.stdout:3766: done (0 left) 2024-09-13T14:31:49.041 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 362 2024-09-13T14:31:49.041 INFO:tasks.rados.rados.0.smithi026.stdout:3767: seq_num 1180 ranges {3973120=565248} 2024-09-13T14:31:49.065 INFO:tasks.rados.rados.0.smithi026.stdout:3767: writing smithi02683976-31 from 3973120 to 4538368 tid 1 2024-09-13T14:31:49.065 INFO:tasks.rados.rados.0.smithi026.stdout:3768: delete oid 30 current snap is 362 2024-09-13T14:31:49.072 INFO:tasks.rados.rados.0.smithi026.stdout:3768: done (1 left) 2024-09-13T14:31:49.072 INFO:tasks.rados.rados.0.smithi026.stdout:3769: delete oid 22 current snap is 362 2024-09-13T14:31:49.075 INFO:tasks.rados.rados.0.smithi026.stdout:3767: finishing write tid 1 to smithi02683976-31 2024-09-13T14:31:49.075 INFO:tasks.rados.rados.0.smithi026.stdout:3767: finishing write tid 2 to smithi02683976-31 2024-09-13T14:31:49.077 INFO:tasks.rados.rados.0.smithi026.stdout:3767: finishing write tid 3 to smithi02683976-31 2024-09-13T14:31:49.077 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 2009 (ObjNum 1180 snap 362 seq_num 1180) dirty exists 2024-09-13T14:31:49.077 INFO:tasks.rados.rados.0.smithi026.stdout:3767: left oid 31 (ObjNum 1180 snap 362 seq_num 1180) 2024-09-13T14:31:49.078 INFO:tasks.rados.rados.0.smithi026.stdout:3767: done (1 left) 2024-09-13T14:31:49.078 INFO:tasks.rados.rados.0.smithi026.stdout:3769: done (0 left) 2024-09-13T14:31:49.078 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 362 2024-09-13T14:31:49.078 INFO:tasks.rados.rados.0.smithi026.stdout:3770: seq_num 1183 ranges {2039808=622592} 2024-09-13T14:31:49.093 INFO:tasks.rados.rados.0.smithi026.stdout:3770: writing smithi02683976-44 from 2039808 to 2662400 tid 1 2024-09-13T14:31:49.093 INFO:tasks.rados.rados.0.smithi026.stdout:3771: snap_remove snap 351 2024-09-13T14:31:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:49 smithi026 ceph-mon[97022]: osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:31:49.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:49 smithi026 ceph-mon[97022]: osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:31:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:49 smithi026 ceph-mon[99461]: osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:31:49.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:49 smithi026 ceph-mon[99461]: osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:31:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:49 smithi145 ceph-mon[89145]: osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:31:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:49 smithi145 ceph-mon[89145]: osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:31:50.099 INFO:tasks.rados.rados.0.smithi026.stdout:3770: finishing write tid 1 to smithi02683976-44 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3770: finishing write tid 2 to smithi02683976-44 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3770: finishing write tid 3 to smithi02683976-44 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1660 (ObjNum 1183 snap 362 seq_num 1183) dirty exists 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3770: left oid 44 (ObjNum 1183 snap 362 seq_num 1183) 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3770: done (1 left) 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3771: done (0 left) 2024-09-13T14:31:50.100 INFO:tasks.rados.rados.0.smithi026.stdout:3772: snap_create 2024-09-13T14:31:50.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:50 smithi026 ceph-mon[97022]: pgmap v798: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 108 KiB/s wr, 5 op/s 2024-09-13T14:31:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:50 smithi026 ceph-mon[99461]: pgmap v798: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 108 KiB/s wr, 5 op/s 2024-09-13T14:31:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:50 smithi145 ceph-mon[89145]: pgmap v798: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 108 KiB/s wr, 5 op/s 2024-09-13T14:31:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:31:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:31:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:51.111 INFO:tasks.rados.rados.0.smithi026.stdout:3772: done (0 left) 2024-09-13T14:31:51.111 INFO:tasks.rados.rados.0.smithi026.stdout:3773: rollback oid 21 current snap is 363 2024-09-13T14:31:51.111 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 21 to 356 2024-09-13T14:31:51.111 INFO:tasks.rados.rados.0.smithi026.stdout:3774: rollback oid 38 current snap is 363 2024-09-13T14:31:51.112 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 38 to 341 2024-09-13T14:31:51.112 INFO:tasks.rados.rados.0.smithi026.stdout:3775: snap_create 2024-09-13T14:31:51.115 INFO:tasks.rados.rados.0.smithi026.stdout:3774: finishing rollback tid 0 to smithi02683976-38 2024-09-13T14:31:51.117 INFO:tasks.rados.rados.0.smithi026.stdout:3774: finishing rollback tid 1 to smithi02683976-38 2024-09-13T14:31:51.117 INFO:tasks.rados.rados.0.smithi026.stdout:3774: finishing rollback tid 2 to smithi02683976-38 2024-09-13T14:31:51.117 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 38 v 2040 (ObjNum 977 snap 306 seq_num 977) dirty exists 2024-09-13T14:31:51.118 INFO:tasks.rados.rados.0.smithi026.stdout:3773: finishing rollback tid 1 to smithi02683976-21 2024-09-13T14:31:51.118 INFO:tasks.rados.rados.0.smithi026.stdout:3773: finishing rollback tid 2 to smithi02683976-21 2024-09-13T14:31:51.119 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 21 v 2019 (ObjNum 1104 snap 334 seq_num 1104) dirty exists 2024-09-13T14:31:51.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:51 smithi026 ceph-mon[97022]: osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:31:51.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:51 smithi026 ceph-mon[99461]: osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:31:51.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:31:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:31:51] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:31:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:51 smithi145 ceph-mon[89145]: osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3773: done (2 left) 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3774: done (1 left) 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3775: done (0 left) 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3776: read oid 9 snap 345 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3776: expect (ObjNum 1039 snap 319 seq_num 1039) 2024-09-13T14:31:52.112 INFO:tasks.rados.rados.0.smithi026.stdout:3777: delete oid 37 current snap is 364 2024-09-13T14:31:52.115 INFO:tasks.rados.rados.0.smithi026.stdout:3777: done (1 left) 2024-09-13T14:31:52.115 INFO:tasks.rados.rados.0.smithi026.stdout:3778: rollback oid 8 current snap is 364 2024-09-13T14:31:52.115 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 8 to 347 2024-09-13T14:31:52.115 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 364 2024-09-13T14:31:52.115 INFO:tasks.rados.rados.0.smithi026.stdout:3779: seq_num 1185 ranges {0=655360} 2024-09-13T14:31:52.121 INFO:tasks.rados.rados.0.smithi026.stdout:3779: writing smithi02683976-32 from 0 to 655360 tid 1 2024-09-13T14:31:52.121 INFO:tasks.rados.rados.0.smithi026.stdout:3780: rmattr oid 7 current snap is 364 2024-09-13T14:31:52.121 INFO:tasks.rados.rados.0.smithi026.stdout:3781: copy_from oid 43 from oid 41 current snap is 364 2024-09-13T14:31:52.121 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 37 current snap is 364 2024-09-13T14:31:52.121 INFO:tasks.rados.rados.0.smithi026.stdout:3782: seq_num 1186 ranges {0=671744} 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:3782: writing smithi02683976-37 from 0 to 671744 tid 1 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:3783: delete oid 1 current snap is 364 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:3778: finishing rollback tid 1 to smithi02683976-8 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:3778: finishing rollback tid 2 to smithi02683976-8 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 8 v 1532 (ObjNum 1108 snap 335 seq_num 1108) dirty exists 2024-09-13T14:31:52.125 INFO:tasks.rados.rados.0.smithi026.stdout:3776: expect (ObjNum 1039 snap 319 seq_num 1039) 2024-09-13T14:31:52.204 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1617 (ObjNum 1051 snap 323 seq_num 1051) dirty exists 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3776: done (6 left) 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3778: done (5 left) 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3780: done (4 left) 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3783: done (3 left) 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3784: rollback oid 5 current snap is 364 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 5 to 361 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3779: finishing write tid 1 to smithi02683976-32 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3779: finishing write tid 2 to smithi02683976-32 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3779: finishing write tid 3 to smithi02683976-32 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 2042 (ObjNum 1185 snap 364 seq_num 1185) dirty exists 2024-09-13T14:31:52.205 INFO:tasks.rados.rados.0.smithi026.stdout:3779: left oid 32 (ObjNum 1185 snap 364 seq_num 1185) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3782: finishing write tid 1 to smithi02683976-37 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3782: finishing write tid 2 to smithi02683976-37 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3782: finishing write tid 3 to smithi02683976-37 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1492 (ObjNum 1186 snap 364 seq_num 1186) dirty exists 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3782: left oid 37 (ObjNum 1186 snap 364 seq_num 1186) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3779: done (3 left) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3782: done (2 left) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3785: setattr oid 23 current snap is 364 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3786: read oid 27 snap -1 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3786: expect (ObjNum 1056 snap 323 seq_num 1056) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3787: read oid 31 snap -1 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3787: expect (ObjNum 1180 snap 362 seq_num 1180) 2024-09-13T14:31:52.206 INFO:tasks.rados.rados.0.smithi026.stdout:3788: read oid 47 snap -1 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3788: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3789: read oid 10 snap -1 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3789: expect deleted 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3790: delete oid 18 current snap is 364 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3784: finishing rollback tid 0 to smithi02683976-5 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3788: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 2043 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3784: finishing rollback tid 1 to smithi02683976-5 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3784: finishing rollback tid 2 to smithi02683976-5 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2500 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3781: finishing copy_from racing read to smithi02683976-43 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3781: finishing copy_from to smithi02683976-43 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 1943 (ObjNum 1133 snap 344 seq_num 1133) dirty exists 2024-09-13T14:31:52.207 INFO:tasks.rados.rados.0.smithi026.stdout:3787: expect (ObjNum 1180 snap 362 seq_num 1180) 2024-09-13T14:31:52.401 INFO:tasks.rados.rados.0.smithi026.stdout:3786: expect (ObjNum 1056 snap 323 seq_num 1056) 2024-09-13T14:31:52.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:52 smithi026 ceph-mon[99461]: pgmap v801: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 7 op/s 2024-09-13T14:31:52.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:52 smithi026 ceph-mon[99461]: osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:31:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:52 smithi026 ceph-mon[97022]: pgmap v801: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 7 op/s 2024-09-13T14:31:52.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:52 smithi026 ceph-mon[97022]: osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:31:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:52 smithi145 ceph-mon[89145]: pgmap v801: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 7 op/s 2024-09-13T14:31:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:52 smithi145 ceph-mon[89145]: osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3781: done (7 left) 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3784: done (6 left) 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3785: done (5 left) 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3786: done (4 left) 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3787: done (3 left) 2024-09-13T14:31:52.658 INFO:tasks.rados.rados.0.smithi026.stdout:3788: done (2 left) 2024-09-13T14:31:52.659 INFO:tasks.rados.rados.0.smithi026.stdout:3789: done (1 left) 2024-09-13T14:31:52.659 INFO:tasks.rados.rados.0.smithi026.stdout:3790: done (0 left) 2024-09-13T14:31:52.659 INFO:tasks.rados.rados.0.smithi026.stdout:3791: rollback oid 1 current snap is 364 2024-09-13T14:31:52.659 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 1 to 359 2024-09-13T14:31:52.659 INFO:tasks.rados.rados.0.smithi026.stdout:3792: snap_create 2024-09-13T14:31:52.661 INFO:tasks.rados.rados.0.smithi026.stdout:3791: finishing rollback tid 1 to smithi02683976-1 2024-09-13T14:31:52.661 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 1 v 2416 (ObjNum 97 snap 0 seq_num 518825888) dirty dne 2024-09-13T14:31:53.122 INFO:tasks.rados.rados.0.smithi026.stdout:3791: done (1 left) 2024-09-13T14:31:53.123 INFO:tasks.rados.rados.0.smithi026.stdout:3792: done (0 left) 2024-09-13T14:31:53.123 INFO:tasks.rados.rados.0.smithi026.stdout:3793: copy_from oid 10 from oid 7 current snap is 365 2024-09-13T14:31:53.123 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 365 2024-09-13T14:31:53.123 INFO:tasks.rados.rados.0.smithi026.stdout:3794: seq_num 1189 ranges {0=720896} 2024-09-13T14:31:53.127 INFO:tasks.rados.rados.0.smithi026.stdout:3794: writing smithi02683976-35 from 0 to 720896 tid 1 2024-09-13T14:31:53.127 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 15 current snap is 365 2024-09-13T14:31:53.127 INFO:tasks.rados.rados.0.smithi026.stdout:3795: seq_num 1190 ranges {180224=737280} 2024-09-13T14:31:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:3795: writing smithi02683976-15 from 180224 to 917504 tid 1 2024-09-13T14:31:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 365 2024-09-13T14:31:53.133 INFO:tasks.rados.rados.0.smithi026.stdout:3796: seq_num 1191 ranges {2940928=753664} 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:3796: writing smithi02683976-5 from 2940928 to 3694592 tid 1 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:3797: read oid 47 snap -1 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:3797: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 31 current snap is 365 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:3794: finishing write tid 1 to smithi02683976-35 2024-09-13T14:31:53.154 INFO:tasks.rados.rados.0.smithi026.stdout:3798: seq_num 1192 ranges {4538368=770048} 2024-09-13T14:31:53.183 INFO:tasks.rados.rados.0.smithi026.stdout:3798: writing smithi02683976-31 from 4538368 to 5308416 tid 1 2024-09-13T14:31:53.184 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 33 current snap is 365 2024-09-13T14:31:53.184 INFO:tasks.rados.rados.0.smithi026.stdout:3799: seq_num 1193 ranges {3801088=786432} 2024-09-13T14:31:53.209 INFO:tasks.rados.rados.0.smithi026.stdout:3799: writing smithi02683976-33 from 3801088 to 4587520 tid 1 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3800: rmattr oid 39 current snap is 365 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3800: done (7 left) 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3801: read oid 42 snap -1 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3801: expect (ObjNum 1145 snap 348 seq_num 1145) 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3802: copy_from oid 48 from oid 2 current snap is 365 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3794: finishing write tid 2 to smithi02683976-35 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3803: rmattr oid 43 current snap is 365 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3794: finishing write tid 3 to smithi02683976-35 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2504 (ObjNum 1189 snap 365 seq_num 1189) dirty exists 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3794: left oid 35 (ObjNum 1189 snap 365 seq_num 1189) 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3793: finishing copy_from racing read to smithi02683976-10 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:3793: finishing copy_from to smithi02683976-10 2024-09-13T14:31:53.210 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 1617 (ObjNum 1051 snap 323 seq_num 1051) dirty exists 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:3795: finishing write tid 1 to smithi02683976-15 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:3795: finishing write tid 2 to smithi02683976-15 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:3795: finishing write tid 3 to smithi02683976-15 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 15 v 1858 (ObjNum 1190 snap 365 seq_num 1190) dirty exists 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:3795: left oid 15 (ObjNum 1190 snap 365 seq_num 1190) 2024-09-13T14:31:53.211 INFO:tasks.rados.rados.0.smithi026.stdout:3797: expect (ObjNum 1065 snap 329 seq_num 1065) 2024-09-13T14:31:53.242 INFO:tasks.rados.rados.0.smithi026.stdout:3796: finishing write tid 1 to smithi02683976-5 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3796: finishing write tid 2 to smithi02683976-5 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3793: done (9 left) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3794: done (8 left) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3795: done (7 left) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3797: done (6 left) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3804: read oid 3 snap -1 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3804: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3805: delete oid 10 current snap is 365 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3796: finishing write tid 3 to smithi02683976-5 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2507 (ObjNum 1191 snap 365 seq_num 1191) dirty exists 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3796: left oid 5 (ObjNum 1191 snap 365 seq_num 1191) 2024-09-13T14:31:53.243 INFO:tasks.rados.rados.0.smithi026.stdout:3798: finishing write tid 1 to smithi02683976-31 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3798: finishing write tid 2 to smithi02683976-31 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3798: finishing write tid 3 to smithi02683976-31 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 2022 (ObjNum 1192 snap 365 seq_num 1192) dirty exists 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3798: left oid 31 (ObjNum 1192 snap 365 seq_num 1192) 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2059 (ObjNum 1133 snap 344 seq_num 1133) dirty exists 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3802: finishing copy_from racing read to smithi02683976-48 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3802: finishing copy_from to smithi02683976-48 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 2026 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3799: finishing write tid 1 to smithi02683976-33 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3799: finishing write tid 2 to smithi02683976-33 2024-09-13T14:31:53.244 INFO:tasks.rados.rados.0.smithi026.stdout:3801: expect (ObjNum 1145 snap 348 seq_num 1145) 2024-09-13T14:31:53.306 INFO:tasks.rados.rados.0.smithi026.stdout:3799: finishing write tid 3 to smithi02683976-33 2024-09-13T14:31:53.306 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2510 (ObjNum 1193 snap 365 seq_num 1193) dirty exists 2024-09-13T14:31:53.306 INFO:tasks.rados.rados.0.smithi026.stdout:3799: left oid 33 (ObjNum 1193 snap 365 seq_num 1193) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3796: done (7 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3798: done (6 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3799: done (5 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3801: done (4 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3802: done (3 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3803: done (2 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3805: done (1 left) 2024-09-13T14:31:53.307 INFO:tasks.rados.rados.0.smithi026.stdout:3806: snap_remove snap 350 2024-09-13T14:31:53.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:53 smithi026 ceph-mon[97022]: osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:31:53.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:53 smithi026 ceph-mon[99461]: osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:31:53.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:53 smithi145 ceph-mon[89145]: osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:31:54.043 INFO:tasks.rados.rados.0.smithi026.stdout:3806: done (1 left) 2024-09-13T14:31:54.043 INFO:tasks.rados.rados.0.smithi026.stdout:3807: snap_remove snap 363 2024-09-13T14:31:54.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[97022]: pgmap v804: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 6 op/s 2024-09-13T14:31:54.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[97022]: osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:31:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[97022]: osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:31:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[99461]: pgmap v804: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 6 op/s 2024-09-13T14:31:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[99461]: osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:31:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:54 smithi026 ceph-mon[99461]: osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:31:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:54 smithi145 ceph-mon[89145]: pgmap v804: 121 pgs: 121 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 654 KiB/s wr, 6 op/s 2024-09-13T14:31:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:54 smithi145 ceph-mon[89145]: osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:31:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:54 smithi145 ceph-mon[89145]: osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:31:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:3807: done (1 left) 2024-09-13T14:31:55.044 INFO:tasks.rados.rados.0.smithi026.stdout:3808: snap_remove snap 364 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3808: done (1 left) 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3809: read oid 10 snap 362 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3809: expect deleted 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3810: read oid 5 snap -1 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3810: expect (ObjNum 1191 snap 365 seq_num 1191) 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3811: read oid 30 snap 359 2024-09-13T14:31:56.051 INFO:tasks.rados.rados.0.smithi026.stdout:3811: expect (ObjNum 1138 snap 345 seq_num 1138) 2024-09-13T14:31:56.052 INFO:tasks.rados.rados.0.smithi026.stdout:3812: read oid 34 snap -1 2024-09-13T14:31:56.052 INFO:tasks.rados.rados.0.smithi026.stdout:3812: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:31:56.052 INFO:tasks.rados.rados.0.smithi026.stdout:3813: snap_remove snap 358 2024-09-13T14:31:56.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:56 smithi026 ceph-mon[99461]: pgmap v807: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 326 KiB/s wr, 5 op/s 2024-09-13T14:31:56.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:56 smithi026 ceph-mon[99461]: osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:31:56.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:56 smithi026 ceph-mon[97022]: pgmap v807: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 326 KiB/s wr, 5 op/s 2024-09-13T14:31:56.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:56 smithi026 ceph-mon[97022]: osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:31:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:56 smithi145 ceph-mon[89145]: pgmap v807: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 326 KiB/s wr, 5 op/s 2024-09-13T14:31:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:56 smithi145 ceph-mon[89145]: osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:31:57.057 INFO:tasks.rados.rados.0.smithi026.stdout:3813: done (5 left) 2024-09-13T14:31:57.058 INFO:tasks.rados.rados.0.smithi026.stdout:3814: read oid 28 snap 346 2024-09-13T14:31:57.058 INFO:tasks.rados.rados.0.smithi026.stdout:3814: expect deleted 2024-09-13T14:31:57.058 INFO:tasks.rados.rados.0.smithi026.stdout:3815: rollback oid 20 current snap is 365 2024-09-13T14:31:57.058 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 356 2024-09-13T14:31:57.058 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 44 current snap is 365 2024-09-13T14:31:57.059 INFO:tasks.rados.rados.0.smithi026.stdout:3816: seq_num 1195 ranges {2662400=417792,3080192=401408} 2024-09-13T14:31:57.076 INFO:tasks.rados.rados.0.smithi026.stdout:3816: writing smithi02683976-44 from 2662400 to 3080192 tid 1 2024-09-13T14:31:57.079 INFO:tasks.rados.rados.0.smithi026.stdout:3816: writing smithi02683976-44 from 3080192 to 3481600 tid 2 2024-09-13T14:31:57.079 INFO:tasks.rados.rados.0.smithi026.stdout:3817: snap_create 2024-09-13T14:31:57.079 INFO:tasks.rados.rados.0.smithi026.stdout:3804: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:31:57.309 INFO:tasks.rados.rados.0.smithi026.stdout:3810: expect (ObjNum 1191 snap 365 seq_num 1191) 2024-09-13T14:31:57.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:57 smithi026 ceph-mon[99461]: osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:31:57.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:57 smithi026 ceph-mon[97022]: osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:31:57.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:57 smithi145 ceph-mon[89145]: osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:31:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.538 INFO:tasks.rados.rados.0.smithi026.stdout:3811: expect (ObjNum 1138 snap 345 seq_num 1138) 2024-09-13T14:31:57.745 INFO:tasks.rados.rados.0.smithi026.stdout:3812: expect (ObjNum 875 snap 271 seq_num 875) 2024-09-13T14:31:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:58 smithi026 ceph-mon[99461]: pgmap v810: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-09-13T14:31:58.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:58 smithi026 ceph-mon[99461]: osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:31:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:58 smithi026 ceph-mon[97022]: pgmap v810: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-09-13T14:31:58.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:58 smithi026 ceph-mon[97022]: osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:31:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:58 smithi145 ceph-mon[89145]: pgmap v810: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-09-13T14:31:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:58 smithi145 ceph-mon[89145]: osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:31:59.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:59 smithi026 ceph-mon[97022]: osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:31:59.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:31:59 smithi026 ceph-mon[97022]: osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:31:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:59 smithi026 ceph-mon[99461]: osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:31:59.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:31:59 smithi026 ceph-mon[99461]: osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:31:59.418 INFO:tasks.rados.rados.0.smithi026.stdout:3815: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:31:59.418 INFO:tasks.rados.rados.0.smithi026.stdout:3815: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:31:59.418 INFO:tasks.rados.rados.0.smithi026.stdout:3815: finishing rollback tid 2 to smithi02683976-20 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2062 (ObjNum 1155 snap 355 seq_num 1155) dirty exists 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3816: finishing write tid 1 to smithi02683976-44 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3816: finishing write tid 2 to smithi02683976-44 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3816: finishing write tid 3 to smithi02683976-44 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3816: finishing write tid 4 to smithi02683976-44 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 1663 (ObjNum 1195 snap 365 seq_num 1195) dirty exists 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3816: left oid 44 (ObjNum 1195 snap 365 seq_num 1195) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3804: done (8 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3809: done (7 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3810: done (6 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3811: done (5 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3812: done (4 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3814: done (3 left) 2024-09-13T14:31:59.419 INFO:tasks.rados.rados.0.smithi026.stdout:3815: done (2 left) 2024-09-13T14:31:59.420 INFO:tasks.rados.rados.0.smithi026.stdout:3816: done (1 left) 2024-09-13T14:31:59.420 INFO:tasks.rados.rados.0.smithi026.stdout:3817: done (0 left) 2024-09-13T14:31:59.420 INFO:tasks.rados.rados.0.smithi026.stdout:3818: copy_from oid 13 from oid 29 current snap is 366 2024-09-13T14:31:59.420 INFO:tasks.rados.rados.0.smithi026.stdout:3819: setattr oid 33 current snap is 366 2024-09-13T14:31:59.420 INFO:tasks.rados.rados.0.smithi026.stdout:3820: setattr oid 30 current snap is 366 2024-09-13T14:31:59.421 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 366 2024-09-13T14:31:59.421 INFO:tasks.rados.rados.0.smithi026.stdout:3821: seq_num 1196 ranges {2351104=434176,2785280=401408} 2024-09-13T14:31:59.437 INFO:tasks.rados.rados.0.smithi026.stdout:3821: writing smithi02683976-7 from 2351104 to 2785280 tid 1 2024-09-13T14:31:59.440 INFO:tasks.rados.rados.0.smithi026.stdout:3821: writing smithi02683976-7 from 2785280 to 3186688 tid 2 2024-09-13T14:31:59.440 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 366 2024-09-13T14:31:59.440 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 2546 (ObjNum 1193 snap 365 seq_num 1193) dirty exists 2024-09-13T14:31:59.440 INFO:tasks.rados.rados.0.smithi026.stdout:3822: seq_num 1197 ranges {4669440=450560,5120000=401408} 2024-09-13T14:31:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:59 smithi145 ceph-mon[89145]: osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:31:59.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:31:59 smithi145 ceph-mon[89145]: osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:31:59.468 INFO:tasks.rados.rados.0.smithi026.stdout:3822: writing smithi02683976-27 from 4669440 to 5120000 tid 1 2024-09-13T14:31:59.470 INFO:tasks.rados.rados.0.smithi026.stdout:3822: writing smithi02683976-27 from 5120000 to 5521408 tid 2 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:3819: done (4 left) 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 47 current snap is 366 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1859 (ObjNum 3905 snap 0 seq_num 540652400) dirty exists 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:3818: finishing copy_from racing read to smithi02683976-13 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:3818: finishing copy_from to smithi02683976-13 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 13 v 2051 (ObjNum 1109 snap 335 seq_num 1109) dirty exists 2024-09-13T14:31:59.471 INFO:tasks.rados.rados.0.smithi026.stdout:3823: seq_num 1198 ranges {1040384=466944,1507328=401408} 2024-09-13T14:31:59.478 INFO:tasks.rados.rados.0.smithi026.stdout:3823: writing smithi02683976-47 from 1040384 to 1507328 tid 1 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:3823: writing smithi02683976-47 from 1507328 to 1908736 tid 2 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:3818: done (4 left) 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:3820: done (3 left) 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:3824: read oid 45 snap -1 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:3824: expect deleted 2024-09-13T14:31:59.482 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 366 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3821: finishing write tid 1 to smithi02683976-7 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3821: finishing write tid 2 to smithi02683976-7 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3821: finishing write tid 3 to smithi02683976-7 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3821: finishing write tid 4 to smithi02683976-7 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1620 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3821: left oid 7 (ObjNum 1196 snap 366 seq_num 1196) 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3822: finishing write tid 1 to smithi02683976-27 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3822: finishing write tid 2 to smithi02683976-27 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3822: finishing write tid 3 to smithi02683976-27 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3822: finishing write tid 4 to smithi02683976-27 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2550 (ObjNum 1197 snap 366 seq_num 1197) dirty exists 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3822: left oid 27 (ObjNum 1197 snap 366 seq_num 1197) 2024-09-13T14:31:59.483 INFO:tasks.rados.rados.0.smithi026.stdout:3825: seq_num 1199 ranges {0=483328,483328=401408} 2024-09-13T14:31:59.484 INFO:tasks.rados.rados.0.smithi026.stdout:3825: writing smithi02683976-49 from 0 to 483328 tid 1 2024-09-13T14:31:59.486 INFO:tasks.rados.rados.0.smithi026.stdout:3825: writing smithi02683976-49 from 483328 to 884736 tid 2 2024-09-13T14:31:59.487 INFO:tasks.rados.rados.0.smithi026.stdout:3821: done (4 left) 2024-09-13T14:31:59.487 INFO:tasks.rados.rados.0.smithi026.stdout:3822: done (3 left) 2024-09-13T14:31:59.487 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 5 current snap is 366 2024-09-13T14:31:59.487 INFO:tasks.rados.rados.0.smithi026.stdout:3826: seq_num 1200 ranges {3694592=499712,4194304=401408} 2024-09-13T14:31:59.509 INFO:tasks.rados.rados.0.smithi026.stdout:3826: writing smithi02683976-5 from 3694592 to 4194304 tid 1 2024-09-13T14:31:59.512 INFO:tasks.rados.rados.0.smithi026.stdout:3826: writing smithi02683976-5 from 4194304 to 4595712 tid 2 2024-09-13T14:31:59.512 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 22 current snap is 366 2024-09-13T14:31:59.512 INFO:tasks.rados.rados.0.smithi026.stdout:3827: seq_num 1201 ranges {0=524288,524288=401408} 2024-09-13T14:31:59.516 INFO:tasks.rados.rados.0.smithi026.stdout:3827: writing smithi02683976-22 from 0 to 524288 tid 1 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:3827: writing smithi02683976-22 from 524288 to 925696 tid 2 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:3828: rmattr oid 37 current snap is 366 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:3828: done (5 left) 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 30 current snap is 366 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:3823: finishing write tid 1 to smithi02683976-47 2024-09-13T14:31:59.519 INFO:tasks.rados.rados.0.smithi026.stdout:3823: finishing write tid 2 to smithi02683976-47 2024-09-13T14:31:59.520 INFO:tasks.rados.rados.0.smithi026.stdout:3823: finishing write tid 3 to smithi02683976-47 2024-09-13T14:31:59.520 INFO:tasks.rados.rados.0.smithi026.stdout:3823: finishing write tid 4 to smithi02683976-47 2024-09-13T14:31:59.520 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 47 v 2037 (ObjNum 1198 snap 366 seq_num 1198) dirty exists 2024-09-13T14:31:59.520 INFO:tasks.rados.rados.0.smithi026.stdout:3823: left oid 47 (ObjNum 1198 snap 366 seq_num 1198) 2024-09-13T14:31:59.520 INFO:tasks.rados.rados.0.smithi026.stdout:3829: seq_num 1202 ranges {0=540672,540672=401408} 2024-09-13T14:31:59.522 INFO:tasks.rados.rados.0.smithi026.stdout:3829: writing smithi02683976-30 from 0 to 540672 tid 1 2024-09-13T14:31:59.524 INFO:tasks.rados.rados.0.smithi026.stdout:3829: writing smithi02683976-30 from 540672 to 942080 tid 2 2024-09-13T14:31:59.524 INFO:tasks.rados.rados.0.smithi026.stdout:3823: done (5 left) 2024-09-13T14:31:59.524 INFO:tasks.rados.rados.0.smithi026.stdout:3824: done (4 left) 2024-09-13T14:31:59.524 INFO:tasks.rados.rados.0.smithi026.stdout:3830: snap_create 2024-09-13T14:31:59.524 INFO:tasks.rados.rados.0.smithi026.stdout:3825: finishing write tid 1 to smithi02683976-49 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3825: finishing write tid 2 to smithi02683976-49 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3825: finishing write tid 3 to smithi02683976-49 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3825: finishing write tid 4 to smithi02683976-49 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2041 (ObjNum 1199 snap 366 seq_num 1199) dirty exists 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3825: left oid 49 (ObjNum 1199 snap 366 seq_num 1199) 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3826: finishing write tid 1 to smithi02683976-5 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3826: finishing write tid 2 to smithi02683976-5 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3826: finishing write tid 3 to smithi02683976-5 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3826: finishing write tid 4 to smithi02683976-5 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2554 (ObjNum 1200 snap 366 seq_num 1200) dirty exists 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3826: left oid 5 (ObjNum 1200 snap 366 seq_num 1200) 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3827: finishing write tid 1 to smithi02683976-22 2024-09-13T14:31:59.525 INFO:tasks.rados.rados.0.smithi026.stdout:3827: finishing write tid 2 to smithi02683976-22 2024-09-13T14:31:59.526 INFO:tasks.rados.rados.0.smithi026.stdout:3827: finishing write tid 3 to smithi02683976-22 2024-09-13T14:31:59.526 INFO:tasks.rados.rados.0.smithi026.stdout:3827: finishing write tid 4 to smithi02683976-22 2024-09-13T14:31:59.526 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2038 (ObjNum 1201 snap 366 seq_num 1201) dirty exists 2024-09-13T14:31:59.526 INFO:tasks.rados.rados.0.smithi026.stdout:3827: left oid 22 (ObjNum 1201 snap 366 seq_num 1201) 2024-09-13T14:31:59.526 INFO:tasks.rados.rados.0.smithi026.stdout:3829: finishing write tid 1 to smithi02683976-30 2024-09-13T14:31:59.527 INFO:tasks.rados.rados.0.smithi026.stdout:3829: finishing write tid 2 to smithi02683976-30 2024-09-13T14:31:59.527 INFO:tasks.rados.rados.0.smithi026.stdout:3829: finishing write tid 3 to smithi02683976-30 2024-09-13T14:31:59.528 INFO:tasks.rados.rados.0.smithi026.stdout:3829: finishing write tid 4 to smithi02683976-30 2024-09-13T14:31:59.528 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1862 (ObjNum 1202 snap 366 seq_num 1202) dirty exists 2024-09-13T14:31:59.528 INFO:tasks.rados.rados.0.smithi026.stdout:3829: left oid 30 (ObjNum 1202 snap 366 seq_num 1202) 2024-09-13T14:32:00.070 INFO:tasks.rados.rados.0.smithi026.stdout:3825: done (4 left) 2024-09-13T14:32:00.070 INFO:tasks.rados.rados.0.smithi026.stdout:3826: done (3 left) 2024-09-13T14:32:00.071 INFO:tasks.rados.rados.0.smithi026.stdout:3827: done (2 left) 2024-09-13T14:32:00.071 INFO:tasks.rados.rados.0.smithi026.stdout:3829: done (1 left) 2024-09-13T14:32:00.071 INFO:tasks.rados.rados.0.smithi026.stdout:3830: done (0 left) 2024-09-13T14:32:00.071 INFO:tasks.rados.rados.0.smithi026.stdout:3831: snap_remove snap 327 2024-09-13T14:32:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:00 smithi026 ceph-mon[97022]: pgmap v813: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 348 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-09-13T14:32:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:00 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:00 smithi026 ceph-mon[99461]: pgmap v813: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 348 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-09-13T14:32:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:00 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:00 smithi145 ceph-mon[89145]: pgmap v813: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 348 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-09-13T14:32:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:00 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:01.076 INFO:tasks.rados.rados.0.smithi026.stdout:3831: done (0 left) 2024-09-13T14:32:01.076 INFO:tasks.rados.rados.0.smithi026.stdout:3832: copy_from oid 49 from oid 2 current snap is 367 2024-09-13T14:32:01.076 INFO:tasks.rados.rados.0.smithi026.stdout:3833: read oid 18 snap -1 2024-09-13T14:32:01.076 INFO:tasks.rados.rados.0.smithi026.stdout:3833: expect deleted 2024-09-13T14:32:01.077 INFO:tasks.rados.rados.0.smithi026.stdout:3834: copy_from oid 12 from oid 4 current snap is 367 2024-09-13T14:32:01.077 INFO:tasks.rados.rados.0.smithi026.stdout:3835: rollback oid 25 current snap is 367 2024-09-13T14:32:01.077 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 25 to 359 2024-09-13T14:32:01.077 INFO:tasks.rados.rados.0.smithi026.stdout:3836: snap_create 2024-09-13T14:32:01.081 INFO:tasks.rados.rados.0.smithi026.stdout:3834: finishing copy_from to smithi02683976-12 2024-09-13T14:32:01.082 INFO:tasks.rados.rados.0.smithi026.stdout:3834: got expected ENOENT (src dne) 2024-09-13T14:32:01.082 INFO:tasks.rados.rados.0.smithi026.stdout:3834: finishing copy_from racing read to smithi02683976-12 2024-09-13T14:32:01.089 INFO:tasks.rados.rados.0.smithi026.stdout:3835: finishing rollback tid 0 to smithi02683976-25 2024-09-13T14:32:01.090 INFO:tasks.rados.rados.0.smithi026.stdout:3835: finishing rollback tid 1 to smithi02683976-25 2024-09-13T14:32:01.093 INFO:tasks.rados.rados.0.smithi026.stdout:3835: finishing rollback tid 2 to smithi02683976-25 2024-09-13T14:32:01.093 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2030 (ObjNum 1157 snap 355 seq_num 1157) dirty exists 2024-09-13T14:32:01.097 INFO:tasks.rados.rados.0.smithi026.stdout:3832: finishing copy_from racing read to smithi02683976-49 2024-09-13T14:32:01.098 INFO:tasks.rados.rados.0.smithi026.stdout:3832: finishing copy_from to smithi02683976-49 2024-09-13T14:32:01.098 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2026 (ObjNum 1029 snap 316 seq_num 1029) dirty exists 2024-09-13T14:32:01.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:32:01.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:01.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:01.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:32:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:01] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.51.0" 2024-09-13T14:32:02.073 INFO:tasks.rados.rados.0.smithi026.stdout:3832: done (4 left) 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:3833: done (3 left) 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:3834: done (2 left) 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:3835: done (1 left) 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:3836: done (0 left) 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 368 2024-09-13T14:32:02.074 INFO:tasks.rados.rados.0.smithi026.stdout:3837: seq_num 1203 ranges {655360=557056,1212416=401408} 2024-09-13T14:32:02.080 INFO:tasks.rados.rados.0.smithi026.stdout:3837: writing smithi02683976-32 from 655360 to 1212416 tid 1 2024-09-13T14:32:02.083 INFO:tasks.rados.rados.0.smithi026.stdout:3837: writing smithi02683976-32 from 1212416 to 1613824 tid 2 2024-09-13T14:32:02.083 INFO:tasks.rados.rados.0.smithi026.stdout:3838: snap_remove snap 347 2024-09-13T14:32:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[97022]: pgmap v816: 121 pgs: 121 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-13T14:32:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[97022]: osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:32:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[97022]: osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:32:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[99461]: pgmap v816: 121 pgs: 121 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-13T14:32:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[99461]: osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:32:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:02 smithi026 ceph-mon[99461]: osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:32:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:02 smithi145 ceph-mon[89145]: pgmap v816: 121 pgs: 121 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-13T14:32:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:02 smithi145 ceph-mon[89145]: osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:32:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:02 smithi145 ceph-mon[89145]: osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3838: done (1 left) 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3839: copy_from oid 44 from oid 3 current snap is 368 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3837: finishing write tid 1 to smithi02683976-32 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3837: finishing write tid 2 to smithi02683976-32 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3837: finishing write tid 3 to smithi02683976-32 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3837: finishing write tid 4 to smithi02683976-32 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 2047 (ObjNum 1203 snap 368 seq_num 1203) dirty exists 2024-09-13T14:32:03.089 INFO:tasks.rados.rados.0.smithi026.stdout:3837: left oid 32 (ObjNum 1203 snap 368 seq_num 1203) 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3837: done (1 left) 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3840: read oid 40 snap -1 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3840: expect deleted 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3841: copy_from oid 5 from oid 7 current snap is 368 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3842: rmattr oid 13 current snap is 368 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3842: done (3 left) 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3843: read oid 43 snap -1 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3843: expect (ObjNum 1133 snap 344 seq_num 1133) 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3844: read oid 33 snap -1 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3844: expect (ObjNum 1193 snap 365 seq_num 1193) 2024-09-13T14:32:03.090 INFO:tasks.rados.rados.0.smithi026.stdout:3845: snap_remove snap 345 2024-09-13T14:32:03.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:32:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:32:03.101+0000 7ff25c8cf640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1bc after trimming was started 2024-09-13T14:32:04.045 INFO:tasks.rados.rados.0.smithi026.stdout:3844: expect (ObjNum 1193 snap 365 seq_num 1193) 2024-09-13T14:32:04.319 INFO:tasks.rados.rados.0.smithi026.stdout:3841: finishing copy_from racing read to smithi02683976-5 2024-09-13T14:32:04.319 INFO:tasks.rados.rados.0.smithi026.stdout:3841: finishing copy_from to smithi02683976-5 2024-09-13T14:32:04.319 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 1620 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:32:04.319 INFO:tasks.rados.rados.0.smithi026.stdout:3843: expect (ObjNum 1133 snap 344 seq_num 1133) 2024-09-13T14:32:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[97022]: pgmap v819: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2024-09-13T14:32:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[97022]: osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:32:04.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[97022]: osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:32:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[99461]: pgmap v819: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2024-09-13T14:32:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[99461]: osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:32:04.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:04 smithi026 ceph-mon[99461]: osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:32:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:04 smithi145 ceph-mon[89145]: pgmap v819: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.3 MiB/s wr, 21 op/s 2024-09-13T14:32:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:04 smithi145 ceph-mon[89145]: osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:32:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:04 smithi145 ceph-mon[89145]: osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3839: finishing copy_from racing read to smithi02683976-44 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3839: finishing copy_from to smithi02683976-44 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 44 v 2370 (ObjNum 1151 snap 349 seq_num 1151) dirty exists 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3839: done (5 left) 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3840: done (4 left) 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3841: done (3 left) 2024-09-13T14:32:04.648 INFO:tasks.rados.rados.0.smithi026.stdout:3843: done (2 left) 2024-09-13T14:32:04.649 INFO:tasks.rados.rados.0.smithi026.stdout:3844: done (1 left) 2024-09-13T14:32:04.649 INFO:tasks.rados.rados.0.smithi026.stdout:3845: done (0 left) 2024-09-13T14:32:04.649 INFO:tasks.rados.rados.0.smithi026.stdout:3846: setattr oid 36 current snap is 368 2024-09-13T14:32:04.649 INFO:tasks.rados.rados.0.smithi026.stdout:3847: delete oid 33 current snap is 368 2024-09-13T14:32:04.654 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2594 (ObjNum 1098 snap 334 seq_num 1098) dirty exists 2024-09-13T14:32:04.655 INFO:tasks.rados.rados.0.smithi026.stdout:3846: done (1 left) 2024-09-13T14:32:04.655 INFO:tasks.rados.rados.0.smithi026.stdout:3847: done (0 left) 2024-09-13T14:32:04.655 INFO:tasks.rados.rados.0.smithi026.stdout:3848: read oid 23 snap -1 2024-09-13T14:32:04.655 INFO:tasks.rados.rados.0.smithi026.stdout:3848: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:04.655 INFO:tasks.rados.rados.0.smithi026.stdout:3849: snap_remove snap 354 2024-09-13T14:32:05.095 INFO:tasks.rados.rados.0.smithi026.stdout:3849: done (1 left) 2024-09-13T14:32:05.096 INFO:tasks.rados.rados.0.smithi026.stdout:3850: rmattr oid 18 current snap is 368 2024-09-13T14:32:05.096 INFO:tasks.rados.rados.0.smithi026.stdout:3850: done (1 left) 2024-09-13T14:32:05.096 INFO:tasks.rados.rados.0.smithi026.stdout:3851: snap_remove snap 341 2024-09-13T14:32:06.100 INFO:tasks.rados.rados.0.smithi026.stdout:3851: done (1 left) 2024-09-13T14:32:06.100 INFO:tasks.rados.rados.0.smithi026.stdout:3852: snap_create 2024-09-13T14:32:06.100 INFO:tasks.rados.rados.0.smithi026.stdout:3848: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:06.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:06 smithi026 ceph-mon[99461]: pgmap v822: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1010 KiB/s wr, 2 op/s 2024-09-13T14:32:06.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:06 smithi026 ceph-mon[99461]: osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:32:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:06 smithi026 ceph-mon[97022]: pgmap v822: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1010 KiB/s wr, 2 op/s 2024-09-13T14:32:06.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:06 smithi026 ceph-mon[97022]: osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:32:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:06 smithi145 ceph-mon[89145]: pgmap v822: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1010 KiB/s wr, 2 op/s 2024-09-13T14:32:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:06 smithi145 ceph-mon[89145]: osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:32:07.296 INFO:tasks.rados.rados.0.smithi026.stdout:3848: done (1 left) 2024-09-13T14:32:07.296 INFO:tasks.rados.rados.0.smithi026.stdout:3852: done (0 left) 2024-09-13T14:32:07.296 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 29 current snap is 369 2024-09-13T14:32:07.296 INFO:tasks.rados.rados.0.smithi026.stdout:3853: seq_num 1205 ranges {3301376=589824,3891200=401408} 2024-09-13T14:32:07.317 INFO:tasks.rados.rados.0.smithi026.stdout:3853: writing smithi02683976-29 from 3301376 to 3891200 tid 1 2024-09-13T14:32:07.320 INFO:tasks.rados.rados.0.smithi026.stdout:3853: writing smithi02683976-29 from 3891200 to 4292608 tid 2 2024-09-13T14:32:07.320 INFO:tasks.rados.rados.0.smithi026.stdout:3854: delete oid 37 current snap is 369 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3854: done (1 left) 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3855: copy_from oid 33 from oid 15 current snap is 369 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3856: read oid 20 snap 362 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3856: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3857: copy_from oid 39 from oid 7 current snap is 369 2024-09-13T14:32:07.323 INFO:tasks.rados.rados.0.smithi026.stdout:3858: snap_create 2024-09-13T14:32:07.325 INFO:tasks.rados.rados.0.smithi026.stdout:3853: finishing write tid 1 to smithi02683976-29 2024-09-13T14:32:07.326 INFO:tasks.rados.rados.0.smithi026.stdout:3853: finishing write tid 2 to smithi02683976-29 2024-09-13T14:32:07.327 INFO:tasks.rados.rados.0.smithi026.stdout:3853: finishing write tid 3 to smithi02683976-29 2024-09-13T14:32:07.330 INFO:tasks.rados.rados.0.smithi026.stdout:3853: finishing write tid 4 to smithi02683976-29 2024-09-13T14:32:07.330 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 29 v 2066 (ObjNum 1205 snap 369 seq_num 1205) dirty exists 2024-09-13T14:32:07.330 INFO:tasks.rados.rados.0.smithi026.stdout:3853: left oid 29 (ObjNum 1205 snap 369 seq_num 1205) 2024-09-13T14:32:07.332 INFO:tasks.rados.rados.0.smithi026.stdout:3856: expect (ObjNum 1029 snap 316 seq_num 1029) 2024-09-13T14:32:07.346 INFO:tasks.rados.rados.0.smithi026.stdout:3855: finishing copy_from racing read to smithi02683976-33 2024-09-13T14:32:07.346 INFO:tasks.rados.rados.0.smithi026.stdout:3855: finishing copy_from to smithi02683976-33 2024-09-13T14:32:07.347 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 33 v 1858 (ObjNum 1190 snap 365 seq_num 1190) dirty exists 2024-09-13T14:32:07.347 INFO:tasks.rados.rados.0.smithi026.stdout:3857: finishing copy_from racing read to smithi02683976-39 2024-09-13T14:32:07.347 INFO:tasks.rados.rados.0.smithi026.stdout:3857: finishing copy_from to smithi02683976-39 2024-09-13T14:32:07.347 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 39 v 1620 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:32:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:07 smithi026 ceph-mon[97022]: osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:32:07.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:07 smithi026 ceph-mon[99461]: osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:32:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:07 smithi145 ceph-mon[89145]: osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:32:08.116 INFO:tasks.rados.rados.0.smithi026.stdout:3853: done (4 left) 2024-09-13T14:32:08.116 INFO:tasks.rados.rados.0.smithi026.stdout:3855: done (3 left) 2024-09-13T14:32:08.116 INFO:tasks.rados.rados.0.smithi026.stdout:3856: done (2 left) 2024-09-13T14:32:08.117 INFO:tasks.rados.rados.0.smithi026.stdout:3857: done (1 left) 2024-09-13T14:32:08.117 INFO:tasks.rados.rados.0.smithi026.stdout:3858: done (0 left) 2024-09-13T14:32:08.117 INFO:tasks.rados.rados.0.smithi026.stdout:3859: delete oid 8 current snap is 370 2024-09-13T14:32:08.121 INFO:tasks.rados.rados.0.smithi026.stdout:3859: done (0 left) 2024-09-13T14:32:08.121 INFO:tasks.rados.rados.0.smithi026.stdout:3860: snap_remove snap 333 2024-09-13T14:32:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:08 smithi026 ceph-mon[97022]: pgmap v825: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-13T14:32:08.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:08 smithi026 ceph-mon[97022]: osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:32:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:08 smithi026 ceph-mon[99461]: pgmap v825: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-13T14:32:08.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:08 smithi026 ceph-mon[99461]: osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:32:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:08 smithi145 ceph-mon[89145]: pgmap v825: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-13T14:32:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:08 smithi145 ceph-mon[89145]: osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:32:09.047 INFO:tasks.rados.rados.0.smithi026.stdout:3860: done (0 left) 2024-09-13T14:32:09.047 INFO:tasks.rados.rados.0.smithi026.stdout:3861: rmattr oid 5 current snap is 370 2024-09-13T14:32:09.047 INFO:tasks.rados.rados.0.smithi026.stdout:3862: snap_create 2024-09-13T14:32:09.061 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2619 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:32:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:09 smithi026 ceph-mon[97022]: osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:32:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:09 smithi026 ceph-mon[97022]: osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:32:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:09 smithi026 ceph-mon[99461]: osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:32:09.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:09 smithi026 ceph-mon[99461]: osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:32:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:09 smithi145 ceph-mon[89145]: osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:32:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:09 smithi145 ceph-mon[89145]: osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:32:10.047 INFO:tasks.rados.rados.0.smithi026.stdout:3861: done (1 left) 2024-09-13T14:32:10.048 INFO:tasks.rados.rados.0.smithi026.stdout:3862: done (0 left) 2024-09-13T14:32:10.048 INFO:tasks.rados.rados.0.smithi026.stdout:3863: snap_remove snap 342 2024-09-13T14:32:10.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:10 smithi026 ceph-mon[99461]: pgmap v828: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T14:32:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:10 smithi026 ceph-mon[99461]: osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:32:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:10 smithi026 ceph-mon[97022]: pgmap v828: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T14:32:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:10 smithi026 ceph-mon[97022]: osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:32:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:10 smithi145 ceph-mon[89145]: pgmap v828: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 292 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-13T14:32:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:10 smithi145 ceph-mon[89145]: osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:32:10.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:11.051 INFO:tasks.rados.rados.0.smithi026.stdout:3863: done (0 left) 2024-09-13T14:32:11.051 INFO:tasks.rados.rados.0.smithi026.stdout:3864: snap_remove snap 359 2024-09-13T14:32:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:11] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:32:12.055 INFO:tasks.rados.rados.0.smithi026.stdout:3864: done (0 left) 2024-09-13T14:32:12.055 INFO:tasks.rados.rados.0.smithi026.stdout:3865: snap_create 2024-09-13T14:32:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[99461]: pgmap v831: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 224 KiB/s wr, 2 op/s 2024-09-13T14:32:12.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[99461]: osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:32:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[97022]: pgmap v831: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 224 KiB/s wr, 2 op/s 2024-09-13T14:32:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[97022]: osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:32:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:12 smithi145 ceph-mon[89145]: pgmap v831: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 255 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 224 KiB/s wr, 2 op/s 2024-09-13T14:32:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:12 smithi145 ceph-mon[89145]: osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:32:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:13.061 INFO:tasks.rados.rados.0.smithi026.stdout:3865: done (0 left) 2024-09-13T14:32:13.061 INFO:tasks.rados.rados.0.smithi026.stdout:3866: read oid 36 snap 367 2024-09-13T14:32:13.061 INFO:tasks.rados.rados.0.smithi026.stdout:3866: expect (ObjNum 1098 snap 334 seq_num 1098) 2024-09-13T14:32:13.061 INFO:tasks.rados.rados.0.smithi026.stdout:3867: setattr oid 19 current snap is 372 2024-09-13T14:32:13.062 INFO:tasks.rados.rados.0.smithi026.stdout:3868: copy_from oid 31 from oid 11 current snap is 372 2024-09-13T14:32:13.063 INFO:tasks.rados.rados.0.smithi026.stdout:3869: snap_remove snap 360 2024-09-13T14:32:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:13 smithi026 ceph-mon[97022]: osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:32:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:13 smithi026 ceph-mon[99461]: osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:32:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:13 smithi145 ceph-mon[89145]: osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:32:14.049 INFO:tasks.rados.rados.0.smithi026.stdout:3869: done (3 left) 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3870: rmattr oid 15 current snap is 372 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3870: done (3 left) 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3871: rollback oid 12 current snap is 372 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 12 to 361 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2090 (ObjNum 1159 snap 358 seq_num 1159) dirty exists 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3868: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3868: finishing copy_from to smithi02683976-31 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 1934 (ObjNum 1156 snap 355 seq_num 1156) dirty exists 2024-09-13T14:32:14.050 INFO:tasks.rados.rados.0.smithi026.stdout:3866: expect (ObjNum 1098 snap 334 seq_num 1098) 2024-09-13T14:32:14.307 INFO:tasks.rados.rados.0.smithi026.stdout:3871: finishing rollback tid 1 to smithi02683976-12 2024-09-13T14:32:14.307 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 12 v 2032 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-13T14:32:14.308 INFO:tasks.rados.rados.0.smithi026.stdout:3866: done (3 left) 2024-09-13T14:32:14.308 INFO:tasks.rados.rados.0.smithi026.stdout:3867: done (2 left) 2024-09-13T14:32:14.308 INFO:tasks.rados.rados.0.smithi026.stdout:3868: done (1 left) 2024-09-13T14:32:14.308 INFO:tasks.rados.rados.0.smithi026.stdout:3871: done (0 left) 2024-09-13T14:32:14.308 INFO:tasks.rados.rados.0.smithi026.stdout:3872: snap_remove snap 371 2024-09-13T14:32:14.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[99461]: pgmap v834: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 249 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-13T14:32:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[99461]: osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:32:14.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[99461]: osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:32:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[97022]: pgmap v834: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 249 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-13T14:32:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[97022]: osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:32:14.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:14 smithi026 ceph-mon[97022]: osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:32:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:14 smithi145 ceph-mon[89145]: pgmap v834: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 249 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-09-13T14:32:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:14 smithi145 ceph-mon[89145]: osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:32:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:14 smithi145 ceph-mon[89145]: osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:32:15.069 INFO:tasks.rados.rados.0.smithi026.stdout:3872: done (0 left) 2024-09-13T14:32:15.069 INFO:tasks.rados.rados.0.smithi026.stdout:3873: snap_remove snap 353 2024-09-13T14:32:16.071 INFO:tasks.rados.rados.0.smithi026.stdout:3873: done (0 left) 2024-09-13T14:32:16.071 INFO:tasks.rados.rados.0.smithi026.stdout:3874: copy_from oid 37 from oid 24 current snap is 372 2024-09-13T14:32:16.071 INFO:tasks.rados.rados.0.smithi026.stdout:3875: copy_from oid 28 from oid 12 current snap is 372 2024-09-13T14:32:16.071 INFO:tasks.rados.rados.0.smithi026.stdout:3876: snap_remove snap 356 2024-09-13T14:32:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:16 smithi026 ceph-mon[99461]: pgmap v837: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 3 op/s 2024-09-13T14:32:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:16 smithi026 ceph-mon[99461]: osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:32:16.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:16 smithi026 ceph-mon[97022]: pgmap v837: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 3 op/s 2024-09-13T14:32:16.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:16 smithi026 ceph-mon[97022]: osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:32:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:16 smithi145 ceph-mon[89145]: pgmap v837: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 1.2 MiB/s wr, 3 op/s 2024-09-13T14:32:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:16 smithi145 ceph-mon[89145]: osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:32:17.084 INFO:tasks.rados.rados.0.smithi026.stdout:3876: done (2 left) 2024-09-13T14:32:17.084 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 372 2024-09-13T14:32:17.084 INFO:tasks.rados.rados.0.smithi026.stdout:3877: seq_num 1208 ranges {6815744=638976,7454720=401408} 2024-09-13T14:32:17.127 INFO:tasks.rados.rados.0.smithi026.stdout:3877: writing smithi02683976-43 from 6815744 to 7454720 tid 1 2024-09-13T14:32:17.130 INFO:tasks.rados.rados.0.smithi026.stdout:3877: writing smithi02683976-43 from 7454720 to 7856128 tid 2 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3875: finishing copy_from to smithi02683976-28 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3875: got expected ENOENT (src dne) 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3875: finishing copy_from racing read to smithi02683976-28 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3874: finishing copy_from to smithi02683976-37 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3874: got expected ENOENT (src dne) 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3874: finishing copy_from racing read to smithi02683976-37 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3874: done (2 left) 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3875: done (1 left) 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3878: rmattr oid 33 current snap is 372 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3878: done (1 left) 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 372 2024-09-13T14:32:17.131 INFO:tasks.rados.rados.0.smithi026.stdout:3879: seq_num 1209 ranges {720896=655360,1376256=401408} 2024-09-13T14:32:17.138 INFO:tasks.rados.rados.0.smithi026.stdout:3879: writing smithi02683976-35 from 720896 to 1376256 tid 1 2024-09-13T14:32:17.141 INFO:tasks.rados.rados.0.smithi026.stdout:3879: writing smithi02683976-35 from 1376256 to 1777664 tid 2 2024-09-13T14:32:17.141 INFO:tasks.rados.rados.0.smithi026.stdout:3880: setattr oid 20 current snap is 372 2024-09-13T14:32:17.141 INFO:tasks.rados.rados.0.smithi026.stdout:3877: finishing write tid 1 to smithi02683976-43 2024-09-13T14:32:17.141 INFO:tasks.rados.rados.0.smithi026.stdout:3877: finishing write tid 2 to smithi02683976-43 2024-09-13T14:32:17.142 INFO:tasks.rados.rados.0.smithi026.stdout:3877: finishing write tid 3 to smithi02683976-43 2024-09-13T14:32:17.142 INFO:tasks.rados.rados.0.smithi026.stdout:3877: finishing write tid 4 to smithi02683976-43 2024-09-13T14:32:17.142 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2128 (ObjNum 1208 snap 372 seq_num 1208) dirty exists 2024-09-13T14:32:17.142 INFO:tasks.rados.rados.0.smithi026.stdout:3877: left oid 43 (ObjNum 1208 snap 372 seq_num 1208) 2024-09-13T14:32:17.143 INFO:tasks.rados.rados.0.smithi026.stdout:3877: done (2 left) 2024-09-13T14:32:17.143 INFO:tasks.rados.rados.0.smithi026.stdout:3881: read oid 29 snap -1 2024-09-13T14:32:17.143 INFO:tasks.rados.rados.0.smithi026.stdout:3881: expect (ObjNum 1205 snap 369 seq_num 1205) 2024-09-13T14:32:17.143 INFO:tasks.rados.rados.0.smithi026.stdout:3882: delete oid 38 current snap is 372 2024-09-13T14:32:17.151 INFO:tasks.rados.rados.0.smithi026.stdout:3879: finishing write tid 1 to smithi02683976-35 2024-09-13T14:32:17.152 INFO:tasks.rados.rados.0.smithi026.stdout:3879: finishing write tid 2 to smithi02683976-35 2024-09-13T14:32:17.152 INFO:tasks.rados.rados.0.smithi026.stdout:3879: finishing write tid 3 to smithi02683976-35 2024-09-13T14:32:17.154 INFO:tasks.rados.rados.0.smithi026.stdout:3882: done (3 left) 2024-09-13T14:32:17.154 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 50 current snap is 372 2024-09-13T14:32:17.154 INFO:tasks.rados.rados.0.smithi026.stdout:3883: seq_num 1211 ranges {2105344=688128,2793472=401408} 2024-09-13T14:32:17.169 INFO:tasks.rados.rados.0.smithi026.stdout:3883: writing smithi02683976-50 from 2105344 to 2793472 tid 1 2024-09-13T14:32:17.172 INFO:tasks.rados.rados.0.smithi026.stdout:3883: writing smithi02683976-50 from 2793472 to 3194880 tid 2 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:3884: rollback oid 22 current snap is 372 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 22 to 365 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:3879: finishing write tid 4 to smithi02683976-35 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2685 (ObjNum 1209 snap 372 seq_num 1209) dirty exists 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:3879: left oid 35 (ObjNum 1209 snap 372 seq_num 1209) 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:3879: done (4 left) 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 28 current snap is 372 2024-09-13T14:32:17.173 INFO:tasks.rados.rados.0.smithi026.stdout:3885: seq_num 1212 ranges {385024=704512,1089536=401408} 2024-09-13T14:32:17.178 INFO:tasks.rados.rados.0.smithi026.stdout:3885: writing smithi02683976-28 from 385024 to 1089536 tid 1 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:3885: writing smithi02683976-28 from 1089536 to 1490944 tid 2 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:3886: read oid 23 snap -1 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:3886: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:3887: rmattr oid 32 current snap is 372 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:3887: done (6 left) 2024-09-13T14:32:17.181 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 40 current snap is 372 2024-09-13T14:32:17.182 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2130 (ObjNum 1155 snap 355 seq_num 1155) dirty exists 2024-09-13T14:32:17.182 INFO:tasks.rados.rados.0.smithi026.stdout:3881: expect (ObjNum 1205 snap 369 seq_num 1205) 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:3884: finishing rollback tid 0 to smithi02683976-22 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:3884: finishing rollback tid 1 to smithi02683976-22 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 22 v 2040 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:3883: finishing write tid 1 to smithi02683976-50 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:3883: finishing write tid 2 to smithi02683976-50 2024-09-13T14:32:17.382 INFO:tasks.rados.rados.0.smithi026.stdout:3888: seq_num 1213 ranges {0=720896,720896=401408} 2024-09-13T14:32:17.385 INFO:tasks.rados.rados.0.smithi026.stdout:3888: writing smithi02683976-40 from 0 to 720896 tid 1 2024-09-13T14:32:17.388 INFO:tasks.rados.rados.0.smithi026.stdout:3888: writing smithi02683976-40 from 720896 to 1122304 tid 2 2024-09-13T14:32:17.388 INFO:tasks.rados.rados.0.smithi026.stdout:3880: done (6 left) 2024-09-13T14:32:17.388 INFO:tasks.rados.rados.0.smithi026.stdout:3881: done (5 left) 2024-09-13T14:32:17.388 INFO:tasks.rados.rados.0.smithi026.stdout:3884: done (4 left) 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 17 current snap is 372 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:3883: finishing write tid 3 to smithi02683976-50 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:3883: finishing write tid 4 to smithi02683976-50 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 50 v 2126 (ObjNum 1211 snap 372 seq_num 1211) dirty exists 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:3883: left oid 50 (ObjNum 1211 snap 372 seq_num 1211) 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:3885: finishing write tid 1 to smithi02683976-28 2024-09-13T14:32:17.389 INFO:tasks.rados.rados.0.smithi026.stdout:3889: seq_num 1214 ranges {2555904=737280,3293184=401408} 2024-09-13T14:32:17.406 INFO:tasks.rados.rados.0.smithi026.stdout:3889: writing smithi02683976-17 from 2555904 to 3293184 tid 1 2024-09-13T14:32:17.408 INFO:tasks.rados.rados.0.smithi026.stdout:3889: writing smithi02683976-17 from 3293184 to 3694592 tid 2 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3883: done (4 left) 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3890: copy_from oid 36 from oid 5 current snap is 372 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3885: finishing write tid 2 to smithi02683976-28 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3885: finishing write tid 3 to smithi02683976-28 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3885: finishing write tid 4 to smithi02683976-28 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 28 v 2044 (ObjNum 1212 snap 372 seq_num 1212) dirty exists 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3885: left oid 28 (ObjNum 1212 snap 372 seq_num 1212) 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3885: done (4 left) 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 7 current snap is 372 2024-09-13T14:32:17.409 INFO:tasks.rados.rados.0.smithi026.stdout:3886: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:17.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:17 smithi026 ceph-mon[99461]: osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:32:17.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:17 smithi026 ceph-mon[97022]: osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:32:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:17 smithi145 ceph-mon[89145]: osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:32:18.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:18 smithi026 ceph-mon[99461]: pgmap v840: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 226 KiB/s wr, 6 op/s 2024-09-13T14:32:18.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:18 smithi026 ceph-mon[99461]: osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:32:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:18 smithi026 ceph-mon[97022]: pgmap v840: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 226 KiB/s wr, 6 op/s 2024-09-13T14:32:18.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:18 smithi026 ceph-mon[97022]: osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:32:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:18 smithi145 ceph-mon[89145]: pgmap v840: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 226 KiB/s wr, 6 op/s 2024-09-13T14:32:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:18 smithi145 ceph-mon[89145]: osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:32:18.596 INFO:tasks.rados.rados.0.smithi026.stdout:3888: finishing write tid 1 to smithi02683976-40 2024-09-13T14:32:18.596 INFO:tasks.rados.rados.0.smithi026.stdout:3888: finishing write tid 2 to smithi02683976-40 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3888: finishing write tid 3 to smithi02683976-40 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3888: finishing write tid 4 to smithi02683976-40 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 40 v 2047 (ObjNum 1213 snap 372 seq_num 1213) dirty exists 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3888: left oid 40 (ObjNum 1213 snap 372 seq_num 1213) 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3889: finishing write tid 1 to smithi02683976-17 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3889: finishing write tid 2 to smithi02683976-17 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3889: finishing write tid 3 to smithi02683976-17 2024-09-13T14:32:18.597 INFO:tasks.rados.rados.0.smithi026.stdout:3891: seq_num 1215 ranges {3186688=753664,3940352=401408} 2024-09-13T14:32:18.617 INFO:tasks.rados.rados.0.smithi026.stdout:3891: writing smithi02683976-7 from 3186688 to 3940352 tid 1 2024-09-13T14:32:18.620 INFO:tasks.rados.rados.0.smithi026.stdout:3891: writing smithi02683976-7 from 3940352 to 4341760 tid 2 2024-09-13T14:32:18.620 INFO:tasks.rados.rados.0.smithi026.stdout:3886: done (4 left) 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3888: done (3 left) 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3892: copy_from oid 27 from oid 24 current snap is 372 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3889: finishing write tid 4 to smithi02683976-17 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 17 v 1951 (ObjNum 1214 snap 372 seq_num 1214) dirty exists 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3889: left oid 17 (ObjNum 1214 snap 372 seq_num 1214) 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3890: finishing copy_from racing read to smithi02683976-36 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3890: finishing copy_from to smithi02683976-36 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2619 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3889: done (3 left) 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3890: done (2 left) 2024-09-13T14:32:18.621 INFO:tasks.rados.rados.0.smithi026.stdout:3893: snap_remove snap 365 2024-09-13T14:32:19.050 INFO:tasks.rados.rados.0.smithi026.stdout:3893: done (2 left) 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3894: rollback oid 5 current snap is 372 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3892: finishing copy_from to smithi02683976-27 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3892: got expected ENOENT (src dne) 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3892: finishing copy_from racing read to smithi02683976-27 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: finishing write tid 1 to smithi02683976-7 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: finishing write tid 2 to smithi02683976-7 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: finishing write tid 3 to smithi02683976-7 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: finishing write tid 4 to smithi02683976-7 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1623 (ObjNum 1215 snap 372 seq_num 1215) dirty exists 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: left oid 7 (ObjNum 1215 snap 372 seq_num 1215) 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 5 to 344 2024-09-13T14:32:19.051 INFO:tasks.rados.rados.0.smithi026.stdout:3891: done (2 left) 2024-09-13T14:32:19.052 INFO:tasks.rados.rados.0.smithi026.stdout:3892: done (1 left) 2024-09-13T14:32:19.052 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 32 current snap is 372 2024-09-13T14:32:19.052 INFO:tasks.rados.rados.0.smithi026.stdout:3895: seq_num 1216 ranges {1613824=770048,2383872=401408} 2024-09-13T14:32:19.065 INFO:tasks.rados.rados.0.smithi026.stdout:3895: writing smithi02683976-32 from 1613824 to 2383872 tid 1 2024-09-13T14:32:19.068 INFO:tasks.rados.rados.0.smithi026.stdout:3895: writing smithi02683976-32 from 2383872 to 2785280 tid 2 2024-09-13T14:32:19.068 INFO:tasks.rados.rados.0.smithi026.stdout:3896: read oid 7 snap 370 2024-09-13T14:32:19.068 INFO:tasks.rados.rados.0.smithi026.stdout:3896: expect (ObjNum 1196 snap 366 seq_num 1196) 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3897: read oid 50 snap -1 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3897: expect (ObjNum 1211 snap 372 seq_num 1211) 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3898: copy_from oid 31 from oid 25 current snap is 372 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3894: finishing rollback tid 0 to smithi02683976-5 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3894: finishing rollback tid 1 to smithi02683976-5 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3894: finishing rollback tid 2 to smithi02683976-5 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2691 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3894: done (4 left) 2024-09-13T14:32:19.069 INFO:tasks.rados.rados.0.smithi026.stdout:3899: snap_remove snap 368 2024-09-13T14:32:20.056 INFO:tasks.rados.rados.0.smithi026.stdout:3899: done (4 left) 2024-09-13T14:32:20.056 INFO:tasks.rados.rados.0.smithi026.stdout:3895: finishing write tid 1 to smithi02683976-32 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:3895: finishing write tid 2 to smithi02683976-32 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:3895: finishing write tid 3 to smithi02683976-32 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:3895: finishing write tid 4 to smithi02683976-32 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 2069 (ObjNum 1216 snap 372 seq_num 1216) dirty exists 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:3895: left oid 32 (ObjNum 1216 snap 372 seq_num 1216) 2024-09-13T14:32:20.057 INFO:tasks.rados.rados.0.smithi026.stdout:3897: expect (ObjNum 1211 snap 372 seq_num 1211) 2024-09-13T14:32:20.185 INFO:tasks.rados.rados.0.smithi026.stdout:3896: expect (ObjNum 1196 snap 366 seq_num 1196) 2024-09-13T14:32:20.356 INFO:tasks.rados.rados.0.smithi026.stdout:3898: finishing copy_from racing read to smithi02683976-31 2024-09-13T14:32:20.356 INFO:tasks.rados.rados.0.smithi026.stdout:3898: finishing copy_from to smithi02683976-31 2024-09-13T14:32:20.356 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 2030 (ObjNum 1157 snap 355 seq_num 1157) dirty exists 2024-09-13T14:32:20.356 INFO:tasks.rados.rados.0.smithi026.stdout:3900: read oid 22 snap 361 2024-09-13T14:32:20.356 INFO:tasks.rados.rados.0.smithi026.stdout:3900: expect (ObjNum 1140 snap 347 seq_num 1140) 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3895: done (4 left) 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3896: done (3 left) 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3897: done (2 left) 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3898: done (1 left) 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3901: read oid 4 snap -1 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3901: expect deleted 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 35 current snap is 372 2024-09-13T14:32:20.357 INFO:tasks.rados.rados.0.smithi026.stdout:3902: seq_num 1217 ranges {1777664=786432,2564096=401408} 2024-09-13T14:32:20.369 INFO:tasks.rados.rados.0.smithi026.stdout:3902: writing smithi02683976-35 from 1777664 to 2564096 tid 1 2024-09-13T14:32:20.372 INFO:tasks.rados.rados.0.smithi026.stdout:3902: writing smithi02683976-35 from 2564096 to 2965504 tid 2 2024-09-13T14:32:20.372 INFO:tasks.rados.rados.0.smithi026.stdout:3903: read oid 27 snap -1 2024-09-13T14:32:20.372 INFO:tasks.rados.rados.0.smithi026.stdout:3903: expect (ObjNum 1197 snap 366 seq_num 1197) 2024-09-13T14:32:20.372 INFO:tasks.rados.rados.0.smithi026.stdout:3904: snap_remove snap 344 2024-09-13T14:32:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:20 smithi026 ceph-mon[97022]: pgmap v842: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 188 KiB/s wr, 5 op/s 2024-09-13T14:32:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:20 smithi026 ceph-mon[97022]: osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:32:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:20 smithi026 ceph-mon[99461]: pgmap v842: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 188 KiB/s wr, 5 op/s 2024-09-13T14:32:20.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:20 smithi026 ceph-mon[99461]: osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:32:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:20 smithi145 ceph-mon[89145]: pgmap v842: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 248 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 188 KiB/s wr, 5 op/s 2024-09-13T14:32:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:20 smithi145 ceph-mon[89145]: osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:32:20.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:21.061 INFO:tasks.rados.rados.0.smithi026.stdout:3902: finishing write tid 1 to smithi02683976-35 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:3902: finishing write tid 2 to smithi02683976-35 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:3902: finishing write tid 3 to smithi02683976-35 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:3902: finishing write tid 4 to smithi02683976-35 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2710 (ObjNum 1217 snap 372 seq_num 1217) dirty exists 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:3902: left oid 35 (ObjNum 1217 snap 372 seq_num 1217) 2024-09-13T14:32:21.062 INFO:tasks.rados.rados.0.smithi026.stdout:3903: expect (ObjNum 1197 snap 366 seq_num 1197) 2024-09-13T14:32:21.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:21 smithi026 ceph-mon[99461]: osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:32:21.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:21 smithi026 ceph-mon[97022]: osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:32:21.428 INFO:tasks.rados.rados.0.smithi026.stdout:3900: expect (ObjNum 1140 snap 347 seq_num 1140) 2024-09-13T14:32:21.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:21 smithi145 ceph-mon[89145]: osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:32:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:21] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:32:22.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:22 smithi026 ceph-mon[99461]: pgmap v845: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 260 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-13T14:32:22.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:22 smithi026 ceph-mon[99461]: osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:32:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:22 smithi026 ceph-mon[97022]: pgmap v845: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 260 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-13T14:32:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:22 smithi026 ceph-mon[97022]: osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:32:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:22 smithi145 ceph-mon[89145]: pgmap v845: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 260 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-13T14:32:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:22 smithi145 ceph-mon[89145]: osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:32:23.630 INFO:tasks.rados.rados.0.smithi026.stdout:3900: done (4 left) 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:3901: done (3 left) 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:3902: done (2 left) 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:3903: done (1 left) 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:3904: done (0 left) 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 372 2024-09-13T14:32:23.631 INFO:tasks.rados.rados.0.smithi026.stdout:3905: seq_num 1218 ranges {442368=802816,1245184=401408} 2024-09-13T14:32:23.637 INFO:tasks.rados.rados.0.smithi026.stdout:3905: writing smithi02683976-49 from 442368 to 1245184 tid 1 2024-09-13T14:32:23.640 INFO:tasks.rados.rados.0.smithi026.stdout:3905: writing smithi02683976-49 from 1245184 to 1646592 tid 2 2024-09-13T14:32:23.640 INFO:tasks.rados.rados.0.smithi026.stdout:3906: delete oid 35 current snap is 372 2024-09-13T14:32:23.643 INFO:tasks.rados.rados.0.smithi026.stdout:3905: finishing write tid 1 to smithi02683976-49 2024-09-13T14:32:23.643 INFO:tasks.rados.rados.0.smithi026.stdout:3905: finishing write tid 2 to smithi02683976-49 2024-09-13T14:32:23.643 INFO:tasks.rados.rados.0.smithi026.stdout:3906: done (1 left) 2024-09-13T14:32:23.643 INFO:tasks.rados.rados.0.smithi026.stdout:3907: delete oid 45 current snap is 372 2024-09-13T14:32:23.643 INFO:tasks.rados.rados.0.smithi026.stdout:3905: finishing write tid 3 to smithi02683976-49 2024-09-13T14:32:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:3907: done (1 left) 2024-09-13T14:32:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:3908: snap_create 2024-09-13T14:32:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:3905: finishing write tid 4 to smithi02683976-49 2024-09-13T14:32:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2050 (ObjNum 1218 snap 372 seq_num 1218) dirty exists 2024-09-13T14:32:23.645 INFO:tasks.rados.rados.0.smithi026.stdout:3905: left oid 49 (ObjNum 1218 snap 372 seq_num 1218) 2024-09-13T14:32:24.050 INFO:tasks.rados.rados.0.smithi026.stdout:3905: done (1 left) 2024-09-13T14:32:24.050 INFO:tasks.rados.rados.0.smithi026.stdout:3908: done (0 left) 2024-09-13T14:32:24.050 INFO:tasks.rados.rados.0.smithi026.stdout:3909: copy_from oid 50 from oid 35 current snap is 373 2024-09-13T14:32:24.050 INFO:tasks.rados.rados.0.smithi026.stdout:3910: snap_create 2024-09-13T14:32:24.055 INFO:tasks.rados.rados.0.smithi026.stdout:3909: finishing copy_from to smithi02683976-50 2024-09-13T14:32:24.056 INFO:tasks.rados.rados.0.smithi026.stdout:3909: got expected ENOENT (src dne) 2024-09-13T14:32:24.056 INFO:tasks.rados.rados.0.smithi026.stdout:3909: finishing copy_from racing read to smithi02683976-50 2024-09-13T14:32:24.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:24 smithi026 ceph-mon[97022]: pgmap v847: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:32:24.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:24 smithi026 ceph-mon[97022]: osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:32:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:24 smithi026 ceph-mon[99461]: pgmap v847: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:32:24.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:24 smithi026 ceph-mon[99461]: osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:32:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:24 smithi145 ceph-mon[89145]: pgmap v847: 121 pgs: 121 active+clean; 261 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-13T14:32:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:24 smithi145 ceph-mon[89145]: osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:32:25.053 INFO:tasks.rados.rados.0.smithi026.stdout:3909: done (1 left) 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3910: done (0 left) 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3911: read oid 23 snap -1 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3911: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3912: copy_from oid 4 from oid 28 current snap is 374 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3913: copy_from oid 30 from oid 7 current snap is 374 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 374 2024-09-13T14:32:25.054 INFO:tasks.rados.rados.0.smithi026.stdout:3914: seq_num 1221 ranges {5521408=458752,5980160=729088,6709248=73728} 2024-09-13T14:32:25.087 INFO:tasks.rados.rados.0.smithi026.stdout:3914: writing smithi02683976-27 from 5521408 to 5980160 tid 1 2024-09-13T14:32:25.091 INFO:tasks.rados.rados.0.smithi026.stdout:3914: writing smithi02683976-27 from 5980160 to 6709248 tid 2 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3914: writing smithi02683976-27 from 6709248 to 6782976 tid 3 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 374 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3912: finishing copy_from racing read to smithi02683976-4 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3912: finishing copy_from to smithi02683976-4 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 4 v 2044 (ObjNum 1212 snap 372 seq_num 1212) dirty exists 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3913: finishing copy_from racing read to smithi02683976-30 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3913: finishing copy_from to smithi02683976-30 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 30 v 1623 (ObjNum 1215 snap 372 seq_num 1215) dirty exists 2024-09-13T14:32:25.092 INFO:tasks.rados.rados.0.smithi026.stdout:3915: seq_num 1222 ranges {442368=475136,917504=606208,1523712=196608} 2024-09-13T14:32:25.097 INFO:tasks.rados.rados.0.smithi026.stdout:3915: writing smithi02683976-48 from 442368 to 917504 tid 1 2024-09-13T14:32:25.101 INFO:tasks.rados.rados.0.smithi026.stdout:3915: writing smithi02683976-48 from 917504 to 1523712 tid 2 2024-09-13T14:32:25.102 INFO:tasks.rados.rados.0.smithi026.stdout:3915: writing smithi02683976-48 from 1523712 to 1720320 tid 3 2024-09-13T14:32:25.102 INFO:tasks.rados.rados.0.smithi026.stdout:3912: done (4 left) 2024-09-13T14:32:25.102 INFO:tasks.rados.rados.0.smithi026.stdout:3913: done (3 left) 2024-09-13T14:32:25.102 INFO:tasks.rados.rados.0.smithi026.stdout:3916: snap_remove snap 369 2024-09-13T14:32:26.068 INFO:tasks.rados.rados.0.smithi026.stdout:3916: done (3 left) 2024-09-13T14:32:26.069 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 14 current snap is 374 2024-09-13T14:32:26.069 INFO:tasks.rados.rados.0.smithi026.stdout:3917: seq_num 1223 ranges {368640=491520,860160=491520,1351680=311296} 2024-09-13T14:32:26.074 INFO:tasks.rados.rados.0.smithi026.stdout:3917: writing smithi02683976-14 from 368640 to 860160 tid 1 2024-09-13T14:32:26.077 INFO:tasks.rados.rados.0.smithi026.stdout:3917: writing smithi02683976-14 from 860160 to 1351680 tid 2 2024-09-13T14:32:26.080 INFO:tasks.rados.rados.0.smithi026.stdout:3917: writing smithi02683976-14 from 1351680 to 1662976 tid 3 2024-09-13T14:32:26.081 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 374 2024-09-13T14:32:26.081 INFO:tasks.rados.rados.0.smithi026.stdout:3918: seq_num 1224 ranges {7856128=507904,8364032=425984,8790016=376832} 2024-09-13T14:32:26.127 INFO:tasks.rados.rados.0.smithi026.stdout:3918: writing smithi02683976-43 from 7856128 to 8364032 tid 1 2024-09-13T14:32:26.129 INFO:tasks.rados.rados.0.smithi026.stdout:3918: writing smithi02683976-43 from 8364032 to 8790016 tid 2 2024-09-13T14:32:26.131 INFO:tasks.rados.rados.0.smithi026.stdout:3918: writing smithi02683976-43 from 8790016 to 9166848 tid 3 2024-09-13T14:32:26.132 INFO:tasks.rados.rados.0.smithi026.stdout:3919: rollback oid 20 current snap is 374 2024-09-13T14:32:26.132 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 20 to 346 2024-09-13T14:32:26.132 INFO:tasks.rados.rados.0.smithi026.stdout:3911: expect (ObjNum 917 snap 286 seq_num 917) 2024-09-13T14:32:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:26 smithi026 ceph-mon[97022]: pgmap v849: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-13T14:32:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:26 smithi026 ceph-mon[97022]: osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:32:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:26 smithi026 ceph-mon[99461]: pgmap v849: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-13T14:32:26.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:26 smithi026 ceph-mon[99461]: osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:32:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:26 smithi145 ceph-mon[89145]: pgmap v849: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 13 op/s 2024-09-13T14:32:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:26 smithi145 ceph-mon[89145]: osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:32:27.320 INFO:tasks.rados.rados.0.smithi026.stdout:3914: finishing write tid 1 to smithi02683976-27 2024-09-13T14:32:27.320 INFO:tasks.rados.rados.0.smithi026.stdout:3914: finishing write tid 2 to smithi02683976-27 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3914: finishing write tid 3 to smithi02683976-27 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3914: finishing write tid 4 to smithi02683976-27 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3914: finishing write tid 5 to smithi02683976-27 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2728 (ObjNum 1221 snap 374 seq_num 1221) dirty exists 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3914: left oid 27 (ObjNum 1221 snap 374 seq_num 1221) 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3915: finishing write tid 1 to smithi02683976-48 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3915: finishing write tid 2 to smithi02683976-48 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3915: finishing write tid 3 to smithi02683976-48 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3911: done (5 left) 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3914: done (4 left) 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3920: read oid 44 snap -1 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:3920: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:32:27.321 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 49 current snap is 374 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3915: finishing write tid 4 to smithi02683976-48 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3915: finishing write tid 5 to smithi02683976-48 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 2041 (ObjNum 1222 snap 374 seq_num 1222) dirty exists 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3915: left oid 48 (ObjNum 1222 snap 374 seq_num 1222) 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: finishing write tid 1 to smithi02683976-14 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: finishing write tid 2 to smithi02683976-14 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: finishing write tid 3 to smithi02683976-14 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: finishing write tid 4 to smithi02683976-14 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: finishing write tid 5 to smithi02683976-14 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 2084 (ObjNum 1223 snap 374 seq_num 1223) dirty exists 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3917: left oid 14 (ObjNum 1223 snap 374 seq_num 1223) 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3918: finishing write tid 1 to smithi02683976-43 2024-09-13T14:32:27.322 INFO:tasks.rados.rados.0.smithi026.stdout:3921: seq_num 1225 ranges {1646592=524288,2170880=712704,2883584=90112} 2024-09-13T14:32:27.332 INFO:tasks.rados.rados.0.smithi026.stdout:3921: writing smithi02683976-49 from 1646592 to 2170880 tid 1 2024-09-13T14:32:27.336 INFO:tasks.rados.rados.0.smithi026.stdout:3921: writing smithi02683976-49 from 2170880 to 2883584 tid 2 2024-09-13T14:32:27.338 INFO:tasks.rados.rados.0.smithi026.stdout:3921: writing smithi02683976-49 from 2883584 to 2973696 tid 3 2024-09-13T14:32:27.338 INFO:tasks.rados.rados.0.smithi026.stdout:3915: done (5 left) 2024-09-13T14:32:27.338 INFO:tasks.rados.rados.0.smithi026.stdout:3917: done (4 left) 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3922: read oid 24 snap -1 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3922: expect deleted 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3923: read oid 29 snap -1 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3923: expect (ObjNum 1205 snap 369 seq_num 1205) 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3924: read oid 28 snap -1 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3924: expect (ObjNum 1212 snap 372 seq_num 1212) 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3925: read oid 19 snap -1 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3925: expect (ObjNum 1159 snap 358 seq_num 1159) 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3926: copy_from oid 14 from oid 35 current snap is 374 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3927: setattr oid 32 current snap is 374 2024-09-13T14:32:27.339 INFO:tasks.rados.rados.0.smithi026.stdout:3918: finishing write tid 2 to smithi02683976-43 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3918: finishing write tid 3 to smithi02683976-43 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3918: finishing write tid 4 to smithi02683976-43 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3919: finishing rollback tid 0 to smithi02683976-20 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3919: finishing rollback tid 1 to smithi02683976-20 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 20 v 2172 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3918: finishing write tid 5 to smithi02683976-43 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2169 (ObjNum 1224 snap 374 seq_num 1224) dirty exists 2024-09-13T14:32:27.340 INFO:tasks.rados.rados.0.smithi026.stdout:3918: left oid 43 (ObjNum 1224 snap 374 seq_num 1224) 2024-09-13T14:32:27.341 INFO:tasks.rados.rados.0.smithi026.stdout:3926: finishing copy_from to smithi02683976-14 2024-09-13T14:32:27.341 INFO:tasks.rados.rados.0.smithi026.stdout:3926: got expected ENOENT (src dne) 2024-09-13T14:32:27.341 INFO:tasks.rados.rados.0.smithi026.stdout:3926: finishing copy_from racing read to smithi02683976-14 2024-09-13T14:32:27.341 INFO:tasks.rados.rados.0.smithi026.stdout:3920: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:32:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:27 smithi026 ceph-mon[97022]: osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:32:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:27 smithi026 ceph-mon[99461]: osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:32:27.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:27 smithi145 ceph-mon[89145]: osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:32:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: finishing write tid 1 to smithi02683976-49 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: finishing write tid 2 to smithi02683976-49 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: finishing write tid 3 to smithi02683976-49 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: finishing write tid 4 to smithi02683976-49 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: finishing write tid 5 to smithi02683976-49 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2054 (ObjNum 1225 snap 374 seq_num 1225) dirty exists 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3921: left oid 49 (ObjNum 1225 snap 374 seq_num 1225) 2024-09-13T14:32:27.572 INFO:tasks.rados.rados.0.smithi026.stdout:3924: expect (ObjNum 1212 snap 372 seq_num 1212) 2024-09-13T14:32:27.617 INFO:tasks.rados.rados.0.smithi026.stdout:3923: expect (ObjNum 1205 snap 369 seq_num 1205) 2024-09-13T14:32:27.807 INFO:tasks.rados.rados.0.smithi026.stdout:3925: expect (ObjNum 1159 snap 358 seq_num 1159) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3918: done (9 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3919: done (8 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3920: done (7 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3921: done (6 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3922: done (5 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3923: done (4 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3924: done (3 left) 2024-09-13T14:32:28.172 INFO:tasks.rados.rados.0.smithi026.stdout:3925: done (2 left) 2024-09-13T14:32:28.173 INFO:tasks.rados.rados.0.smithi026.stdout:3926: done (1 left) 2024-09-13T14:32:28.173 INFO:tasks.rados.rados.0.smithi026.stdout:3928: delete oid 42 current snap is 374 2024-09-13T14:32:28.174 INFO:tasks.rados.rados.0.smithi026.stdout:3928: done (1 left) 2024-09-13T14:32:28.174 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 27 current snap is 374 2024-09-13T14:32:28.174 INFO:tasks.rados.rados.0.smithi026.stdout:3929: seq_num 1227 ranges {6782976=557056,7340032=483328,7823360=319488} 2024-09-13T14:32:28.214 INFO:tasks.rados.rados.0.smithi026.stdout:3929: writing smithi02683976-27 from 6782976 to 7340032 tid 1 2024-09-13T14:32:28.217 INFO:tasks.rados.rados.0.smithi026.stdout:3929: writing smithi02683976-27 from 7340032 to 7823360 tid 2 2024-09-13T14:32:28.219 INFO:tasks.rados.rados.0.smithi026.stdout:3929: writing smithi02683976-27 from 7823360 to 8142848 tid 3 2024-09-13T14:32:28.219 INFO:tasks.rados.rados.0.smithi026.stdout:3930: copy_from oid 8 from oid 37 current snap is 374 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 32 v 2090 (ObjNum 1216 snap 372 seq_num 1216) dirty exists 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:3927: done (2 left) 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:3931: copy_from oid 46 from oid 11 current snap is 374 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:3932: rollback oid 9 current snap is 374 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 9 to 370 2024-09-13T14:32:28.220 INFO:tasks.rados.rados.0.smithi026.stdout:3933: snap_remove snap 372 2024-09-13T14:32:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:28 smithi026 ceph-mon[97022]: pgmap v852: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-13T14:32:28.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:28 smithi026 ceph-mon[99461]: pgmap v852: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-13T14:32:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:28 smithi145 ceph-mon[89145]: pgmap v852: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-13T14:32:29.054 INFO:tasks.rados.rados.0.smithi026.stdout:3933: done (4 left) 2024-09-13T14:32:29.054 INFO:tasks.rados.rados.0.smithi026.stdout:3934: snap_create 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3929: finishing write tid 1 to smithi02683976-27 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3930: finishing copy_from to smithi02683976-8 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3930: got expected ENOENT (src dne) 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3930: finishing copy_from racing read to smithi02683976-8 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3929: finishing write tid 2 to smithi02683976-27 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3929: finishing write tid 3 to smithi02683976-27 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3929: finishing write tid 4 to smithi02683976-27 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3932: finishing rollback tid 0 to smithi02683976-9 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3932: finishing rollback tid 1 to smithi02683976-9 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3932: finishing rollback tid 2 to smithi02683976-9 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 9 v 2151 (ObjNum 1039 snap 319 seq_num 1039) dirty exists 2024-09-13T14:32:29.055 INFO:tasks.rados.rados.0.smithi026.stdout:3929: finishing write tid 5 to smithi02683976-27 2024-09-13T14:32:29.056 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 27 v 2740 (ObjNum 1227 snap 374 seq_num 1227) dirty exists 2024-09-13T14:32:29.056 INFO:tasks.rados.rados.0.smithi026.stdout:3929: left oid 27 (ObjNum 1227 snap 374 seq_num 1227) 2024-09-13T14:32:29.056 INFO:tasks.rados.rados.0.smithi026.stdout:3931: finishing copy_from racing read to smithi02683976-46 2024-09-13T14:32:29.056 INFO:tasks.rados.rados.0.smithi026.stdout:3931: finishing copy_from to smithi02683976-46 2024-09-13T14:32:29.056 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 1934 (ObjNum 1156 snap 355 seq_num 1156) dirty exists 2024-09-13T14:32:30.059 INFO:tasks.rados.rados.0.smithi026.stdout:3929: done (4 left) 2024-09-13T14:32:30.060 INFO:tasks.rados.rados.0.smithi026.stdout:3930: done (3 left) 2024-09-13T14:32:30.060 INFO:tasks.rados.rados.0.smithi026.stdout:3931: done (2 left) 2024-09-13T14:32:30.060 INFO:tasks.rados.rados.0.smithi026.stdout:3932: done (1 left) 2024-09-13T14:32:30.060 INFO:tasks.rados.rados.0.smithi026.stdout:3934: done (0 left) 2024-09-13T14:32:30.061 INFO:tasks.rados.rados.0.smithi026.stdout:3935: setattr oid 5 current snap is 375 2024-09-13T14:32:30.063 INFO:tasks.rados.rados.0.smithi026.stdout:3936: snap_remove snap 370 2024-09-13T14:32:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:30 smithi026 ceph-mon[97022]: pgmap v853: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-13T14:32:30.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:30 smithi026 ceph-mon[97022]: osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:32:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:30 smithi026 ceph-mon[99461]: pgmap v853: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-13T14:32:30.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:30 smithi026 ceph-mon[99461]: osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:32:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:30 smithi145 ceph-mon[89145]: pgmap v853: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-13T14:32:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:30 smithi145 ceph-mon[89145]: osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:32:30.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:31.066 INFO:tasks.rados.rados.0.smithi026.stdout:3936: done (1 left) 2024-09-13T14:32:31.066 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 36 current snap is 375 2024-09-13T14:32:31.066 INFO:tasks.rados.rados.0.smithi026.stdout:3937: seq_num 1228 ranges {3186688=573440,3760128=761856,4521984=40960} 2024-09-13T14:32:31.088 INFO:tasks.rados.rados.0.smithi026.stdout:3937: writing smithi02683976-36 from 3186688 to 3760128 tid 1 2024-09-13T14:32:31.093 INFO:tasks.rados.rados.0.smithi026.stdout:3937: writing smithi02683976-36 from 3760128 to 4521984 tid 2 2024-09-13T14:32:31.094 INFO:tasks.rados.rados.0.smithi026.stdout:3937: writing smithi02683976-36 from 4521984 to 4562944 tid 3 2024-09-13T14:32:31.094 INFO:tasks.rados.rados.0.smithi026.stdout:3938: setattr oid 31 current snap is 375 2024-09-13T14:32:31.094 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 5 v 2744 (ObjNum 1006 snap 311 seq_num 1006) dirty exists 2024-09-13T14:32:31.094 INFO:tasks.rados.rados.0.smithi026.stdout:3937: finishing write tid 1 to smithi02683976-36 2024-09-13T14:32:31.096 INFO:tasks.rados.rados.0.smithi026.stdout:3935: done (2 left) 2024-09-13T14:32:31.096 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 26 current snap is 375 2024-09-13T14:32:31.096 INFO:tasks.rados.rados.0.smithi026.stdout:3939: seq_num 1229 ranges {12730368=589824,13320192=647168,13967360=155648} 2024-09-13T14:32:31.168 INFO:tasks.rados.rados.0.smithi026.stdout:3939: writing smithi02683976-26 from 12730368 to 13320192 tid 1 2024-09-13T14:32:31.172 INFO:tasks.rados.rados.0.smithi026.stdout:3939: writing smithi02683976-26 from 13320192 to 13967360 tid 2 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3939: writing smithi02683976-26 from 13967360 to 14123008 tid 3 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3940: snap_create 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 31 v 2161 (ObjNum 1157 snap 355 seq_num 1157) dirty exists 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3937: finishing write tid 2 to smithi02683976-36 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3937: finishing write tid 3 to smithi02683976-36 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3937: finishing write tid 4 to smithi02683976-36 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3937: finishing write tid 5 to smithi02683976-36 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 36 v 2757 (ObjNum 1228 snap 375 seq_num 1228) dirty exists 2024-09-13T14:32:31.173 INFO:tasks.rados.rados.0.smithi026.stdout:3937: left oid 36 (ObjNum 1228 snap 375 seq_num 1228) 2024-09-13T14:32:31.176 INFO:tasks.rados.rados.0.smithi026.stdout:3939: finishing write tid 1 to smithi02683976-26 2024-09-13T14:32:31.176 INFO:tasks.rados.rados.0.smithi026.stdout:3939: finishing write tid 2 to smithi02683976-26 2024-09-13T14:32:31.178 INFO:tasks.rados.rados.0.smithi026.stdout:3939: finishing write tid 3 to smithi02683976-26 2024-09-13T14:32:31.178 INFO:tasks.rados.rados.0.smithi026.stdout:3939: finishing write tid 4 to smithi02683976-26 2024-09-13T14:32:31.180 INFO:tasks.rados.rados.0.smithi026.stdout:3939: finishing write tid 5 to smithi02683976-26 2024-09-13T14:32:31.180 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 26 v 2044 (ObjNum 1229 snap 375 seq_num 1229) dirty exists 2024-09-13T14:32:31.180 INFO:tasks.rados.rados.0.smithi026.stdout:3939: left oid 26 (ObjNum 1229 snap 375 seq_num 1229) 2024-09-13T14:32:31.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:31 smithi026 ceph-mon[99461]: osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:32:31.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:31 smithi026 ceph-mon[97022]: osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:32:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:31 smithi145 ceph-mon[89145]: osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:32:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:31] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-09-13T14:32:32.066 INFO:tasks.rados.rados.0.smithi026.stdout:3937: done (3 left) 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:3938: done (2 left) 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:3939: done (1 left) 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:3940: done (0 left) 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:3941: rollback oid 46 current snap is 376 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 46 to 362 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 48 current snap is 376 2024-09-13T14:32:32.067 INFO:tasks.rados.rados.0.smithi026.stdout:3942: seq_num 1230 ranges {1720320=606208,2326528=532480,2859008=270336} 2024-09-13T14:32:32.080 INFO:tasks.rados.rados.0.smithi026.stdout:3942: writing smithi02683976-48 from 1720320 to 2326528 tid 1 2024-09-13T14:32:32.083 INFO:tasks.rados.rados.0.smithi026.stdout:3942: writing smithi02683976-48 from 2326528 to 2859008 tid 2 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3942: writing smithi02683976-48 from 2859008 to 3129344 tid 3 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3943: read oid 33 snap -1 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3943: expect (ObjNum 1190 snap 365 seq_num 1190) 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3944: setattr oid 14 current snap is 376 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3941: finishing rollback tid 0 to smithi02683976-46 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3941: finishing rollback tid 1 to smithi02683976-46 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3941: finishing rollback tid 2 to smithi02683976-46 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 46 v 2373 (ObjNum 1139 snap 347 seq_num 1139) dirty exists 2024-09-13T14:32:32.086 INFO:tasks.rados.rados.0.smithi026.stdout:3942: finishing write tid 1 to smithi02683976-48 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3941: done (3 left) 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3945: read oid 3 snap -1 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3945: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3946: read oid 12 snap -1 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3946: expect deleted 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3947: rmattr oid 22 current snap is 376 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3947: done (5 left) 2024-09-13T14:32:32.089 INFO:tasks.rados.rados.0.smithi026.stdout:3948: snap_remove snap 361 2024-09-13T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:32 smithi026 ceph-mon[97022]: pgmap v856: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-13T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:32 smithi026 ceph-mon[97022]: osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:32:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:32 smithi026 ceph-mon[99461]: pgmap v856: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-13T14:32:32.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:32 smithi026 ceph-mon[99461]: osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:32:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:32 smithi145 ceph-mon[89145]: pgmap v856: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-13T14:32:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:32 smithi145 ceph-mon[89145]: osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:32:33.078 INFO:tasks.rados.rados.0.smithi026.stdout:3948: done (5 left) 2024-09-13T14:32:33.078 INFO:tasks.rados.rados.0.smithi026.stdout:3949: snap_remove snap 366 2024-09-13T14:32:33.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:33 smithi026 ceph-mon[97022]: osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:32:33.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:33 smithi026 ceph-mon[99461]: osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:32:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:33 smithi145 ceph-mon[89145]: osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:32:34.054 INFO:tasks.rados.rados.0.smithi026.stdout:3949: done (5 left) 2024-09-13T14:32:34.054 INFO:tasks.rados.rados.0.smithi026.stdout:3950: copy_from oid 18 from oid 32 current snap is 376 2024-09-13T14:32:34.054 INFO:tasks.rados.rados.0.smithi026.stdout:3951: setattr oid 35 current snap is 376 2024-09-13T14:32:34.056 INFO:tasks.rados.rados.0.smithi026.stdout:3952: delete oid 5 current snap is 376 2024-09-13T14:32:34.057 INFO:tasks.rados.rados.0.smithi026.stdout:3942: finishing write tid 2 to smithi02683976-48 2024-09-13T14:32:34.057 INFO:tasks.rados.rados.0.smithi026.stdout:3942: finishing write tid 3 to smithi02683976-48 2024-09-13T14:32:34.057 INFO:tasks.rados.rados.0.smithi026.stdout:3942: finishing write tid 4 to smithi02683976-48 2024-09-13T14:32:34.057 INFO:tasks.rados.rados.0.smithi026.stdout:3943: expect (ObjNum 1190 snap 365 seq_num 1190) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3942: finishing write tid 5 to smithi02683976-48 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 48 v 2045 (ObjNum 1230 snap 376 seq_num 1230) dirty exists 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3942: left oid 48 (ObjNum 1230 snap 376 seq_num 1230) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 14 v 2102 (ObjNum 1223 snap 374 seq_num 1223) dirty exists 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3942: done (7 left) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3943: done (6 left) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3944: done (5 left) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3946: done (4 left) 2024-09-13T14:32:34.099 INFO:tasks.rados.rados.0.smithi026.stdout:3952: done (3 left) 2024-09-13T14:32:34.100 INFO:tasks.rados.rados.0.smithi026.stdout:3953: rollback oid 7 current snap is 376 2024-09-13T14:32:34.100 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 7 to 367 2024-09-13T14:32:34.100 INFO:tasks.rados.rados.0.smithi026.stdout:3945: expect (ObjNum 1151 snap 349 seq_num 1151) 2024-09-13T14:32:34.327 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2772 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3950: finishing copy_from racing read to smithi02683976-18 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3950: finishing copy_from to smithi02683976-18 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 2090 (ObjNum 1216 snap 372 seq_num 1216) dirty exists 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3945: done (3 left) 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3950: done (2 left) 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3951: done (1 left) 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3954: read oid 10 snap -1 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3954: expect deleted 2024-09-13T14:32:34.328 INFO:tasks.rados.rados.0.smithi026.stdout:3955: read oid 1 snap 362 2024-09-13T14:32:34.329 INFO:tasks.rados.rados.0.smithi026.stdout:3955: expect deleted 2024-09-13T14:32:34.329 INFO:tasks.rados.rados.0.smithi026.stdout:3956: snap_remove snap 375 2024-09-13T14:32:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[97022]: pgmap v859: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-13T14:32:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[97022]: osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:32:34.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[97022]: osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:32:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[99461]: pgmap v859: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-13T14:32:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[99461]: osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:32:34.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:34 smithi026 ceph-mon[99461]: osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:32:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:34 smithi145 ceph-mon[89145]: pgmap v859: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-13T14:32:34.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:34 smithi145 ceph-mon[89145]: osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:32:34.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:34 smithi145 ceph-mon[89145]: osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3956: done (3 left) 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3957: copy_from oid 21 from oid 45 current snap is 376 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3953: finishing rollback tid 0 to smithi02683976-7 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3958: copy_from oid 15 from oid 42 current snap is 376 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3953: finishing rollback tid 1 to smithi02683976-7 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3953: finishing rollback tid 2 to smithi02683976-7 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 7 v 1626 (ObjNum 1196 snap 366 seq_num 1196) dirty exists 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3953: done (4 left) 2024-09-13T14:32:35.085 INFO:tasks.rados.rados.0.smithi026.stdout:3954: done (3 left) 2024-09-13T14:32:35.086 INFO:tasks.rados.rados.0.smithi026.stdout:3955: done (2 left) 2024-09-13T14:32:35.086 INFO:tasks.rados.rados.0.smithi026.stdout:3959: delete oid 37 current snap is 376 2024-09-13T14:32:35.091 INFO:tasks.rados.rados.0.smithi026.stdout:3959: done (2 left) 2024-09-13T14:32:35.091 INFO:tasks.rados.rados.0.smithi026.stdout:3960: snap_create 2024-09-13T14:32:35.091 INFO:tasks.rados.rados.0.smithi026.stdout:3957: finishing copy_from to smithi02683976-21 2024-09-13T14:32:35.091 INFO:tasks.rados.rados.0.smithi026.stdout:3957: got expected ENOENT (src dne) 2024-09-13T14:32:35.091 INFO:tasks.rados.rados.0.smithi026.stdout:3957: finishing copy_from racing read to smithi02683976-21 2024-09-13T14:32:35.092 INFO:tasks.rados.rados.0.smithi026.stdout:3958: finishing copy_from to smithi02683976-15 2024-09-13T14:32:35.092 INFO:tasks.rados.rados.0.smithi026.stdout:3958: got expected ENOENT (src dne) 2024-09-13T14:32:35.092 INFO:tasks.rados.rados.0.smithi026.stdout:3958: finishing copy_from racing read to smithi02683976-15 2024-09-13T14:32:36.088 INFO:tasks.rados.rados.0.smithi026.stdout:3957: done (2 left) 2024-09-13T14:32:36.089 INFO:tasks.rados.rados.0.smithi026.stdout:3958: done (1 left) 2024-09-13T14:32:36.089 INFO:tasks.rados.rados.0.smithi026.stdout:3960: done (0 left) 2024-09-13T14:32:36.089 INFO:tasks.rados.rados.0.smithi026.stdout:3961: snap_remove snap 346 2024-09-13T14:32:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:36 smithi026 ceph-mon[97022]: pgmap v862: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 320 KiB/s wr, 8 op/s 2024-09-13T14:32:36.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:36 smithi026 ceph-mon[97022]: osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:32:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:36 smithi026 ceph-mon[99461]: pgmap v862: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 320 KiB/s wr, 8 op/s 2024-09-13T14:32:36.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:36 smithi026 ceph-mon[99461]: osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:32:36.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:36 smithi145 ceph-mon[89145]: pgmap v862: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 320 KiB/s wr, 8 op/s 2024-09-13T14:32:36.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:36 smithi145 ceph-mon[89145]: osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:32:37.090 INFO:tasks.rados.rados.0.smithi026.stdout:3961: done (0 left) 2024-09-13T14:32:37.090 INFO:tasks.rados.rados.0.smithi026.stdout:3962: delete oid 36 current snap is 377 2024-09-13T14:32:37.113 INFO:tasks.rados.rados.0.smithi026.stdout:3962: done (0 left) 2024-09-13T14:32:37.113 INFO:tasks.rados.rados.0.smithi026.stdout:3963: rmattr oid 37 current snap is 377 2024-09-13T14:32:37.113 INFO:tasks.rados.rados.0.smithi026.stdout:3963: done (0 left) 2024-09-13T14:32:37.113 INFO:tasks.rados.rados.0.smithi026.stdout:3964: read oid 40 snap -1 2024-09-13T14:32:37.114 INFO:tasks.rados.rados.0.smithi026.stdout:3964: expect (ObjNum 1213 snap 372 seq_num 1213) 2024-09-13T14:32:37.114 INFO:tasks.rados.rados.0.smithi026.stdout:3965: snap_create 2024-09-13T14:32:37.124 INFO:tasks.rados.rados.0.smithi026.stdout:3964: expect (ObjNum 1213 snap 372 seq_num 1213) 2024-09-13T14:32:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:37 smithi026 ceph-mon[97022]: osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:32:37.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:37 smithi026 ceph-mon[97022]: osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:32:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:37 smithi026 ceph-mon[99461]: osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:32:37.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:37 smithi026 ceph-mon[99461]: osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:32:37.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:37 smithi145 ceph-mon[89145]: osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:32:37.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:37 smithi145 ceph-mon[89145]: osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:32:38.108 INFO:tasks.rados.rados.0.smithi026.stdout:3964: done (1 left) 2024-09-13T14:32:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3965: done (0 left) 2024-09-13T14:32:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3966: setattr oid 19 current snap is 378 2024-09-13T14:32:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3967: copy_from oid 24 from oid 17 current snap is 378 2024-09-13T14:32:38.109 INFO:tasks.rados.rados.0.smithi026.stdout:3968: snap_create 2024-09-13T14:32:38.113 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 19 v 2212 (ObjNum 1159 snap 358 seq_num 1159) dirty exists 2024-09-13T14:32:38.130 INFO:tasks.rados.rados.0.smithi026.stdout:3967: finishing copy_from racing read to smithi02683976-24 2024-09-13T14:32:38.132 INFO:tasks.rados.rados.0.smithi026.stdout:3967: finishing copy_from to smithi02683976-24 2024-09-13T14:32:38.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 24 v 1951 (ObjNum 1214 snap 372 seq_num 1214) dirty exists 2024-09-13T14:32:38.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:38 smithi026 ceph-mon[97022]: pgmap v865: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-09-13T14:32:38.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:38 smithi026 ceph-mon[99461]: pgmap v865: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-09-13T14:32:38.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:38 smithi145 ceph-mon[89145]: pgmap v865: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 282 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-09-13T14:32:39.056 INFO:tasks.rados.rados.0.smithi026.stdout:3966: done (2 left) 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:3967: done (1 left) 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:3968: done (0 left) 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:3969: read oid 6 snap -1 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:3969: expect (ObjNum 1126 snap 340 seq_num 1126) 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 43 current snap is 379 2024-09-13T14:32:39.057 INFO:tasks.rados.rados.0.smithi026.stdout:3970: seq_num 1234 ranges {9166848=671744,9838592=516096,10354688=286720} 2024-09-13T14:32:39.111 INFO:tasks.rados.rados.0.smithi026.stdout:3970: writing smithi02683976-43 from 9166848 to 9838592 tid 1 2024-09-13T14:32:39.114 INFO:tasks.rados.rados.0.smithi026.stdout:3970: writing smithi02683976-43 from 9838592 to 10354688 tid 2 2024-09-13T14:32:39.117 INFO:tasks.rados.rados.0.smithi026.stdout:3970: writing smithi02683976-43 from 10354688 to 10641408 tid 3 2024-09-13T14:32:39.117 INFO:tasks.rados.rados.0.smithi026.stdout:3971: read oid 10 snap -1 2024-09-13T14:32:39.117 INFO:tasks.rados.rados.0.smithi026.stdout:3971: expect deleted 2024-09-13T14:32:39.117 INFO:tasks.rados.rados.0.smithi026.stdout:3972: snap_create 2024-09-13T14:32:39.117 INFO:tasks.rados.rados.0.smithi026.stdout:3969: expect (ObjNum 1126 snap 340 seq_num 1126) 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: finishing write tid 1 to smithi02683976-43 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: finishing write tid 2 to smithi02683976-43 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: finishing write tid 3 to smithi02683976-43 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: finishing write tid 4 to smithi02683976-43 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: finishing write tid 5 to smithi02683976-43 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 43 v 2217 (ObjNum 1234 snap 379 seq_num 1234) dirty exists 2024-09-13T14:32:39.173 INFO:tasks.rados.rados.0.smithi026.stdout:3970: left oid 43 (ObjNum 1234 snap 379 seq_num 1234) 2024-09-13T14:32:39.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:39 smithi026 ceph-mon[97022]: osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:32:39.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:39 smithi026 ceph-mon[97022]: osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:32:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:39 smithi026 ceph-mon[99461]: osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:32:39.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:39 smithi026 ceph-mon[99461]: osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:32:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:39 smithi145 ceph-mon[89145]: osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:32:39.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:39 smithi145 ceph-mon[89145]: osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:32:40.119 INFO:tasks.rados.rados.0.smithi026.stdout:3969: done (3 left) 2024-09-13T14:32:40.120 INFO:tasks.rados.rados.0.smithi026.stdout:3970: done (2 left) 2024-09-13T14:32:40.120 INFO:tasks.rados.rados.0.smithi026.stdout:3971: done (1 left) 2024-09-13T14:32:40.120 INFO:tasks.rados.rados.0.smithi026.stdout:3972: done (0 left) 2024-09-13T14:32:40.120 INFO:tasks.rados.rados.0.smithi026.stdout:3973: setattr oid 34 current snap is 380 2024-09-13T14:32:40.121 INFO:tasks.rados.rados.0.smithi026.stdout:append oid 25 current snap is 380 2024-09-13T14:32:40.121 INFO:tasks.rados.rados.0.smithi026.stdout:3974: seq_num 1235 ranges {8380416=688128,9068544=802816} 2024-09-13T14:32:40.170 INFO:tasks.rados.rados.0.smithi026.stdout:3974: writing smithi02683976-25 from 8380416 to 9068544 tid 1 2024-09-13T14:32:40.175 INFO:tasks.rados.rados.0.smithi026.stdout:3974: writing smithi02683976-25 from 9068544 to 9871360 tid 2 2024-09-13T14:32:40.175 INFO:tasks.rados.rados.0.smithi026.stdout:3975: delete oid 35 current snap is 380 2024-09-13T14:32:40.176 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 34 v 1952 (ObjNum 875 snap 271 seq_num 875) dirty exists 2024-09-13T14:32:40.181 INFO:tasks.rados.rados.0.smithi026.stdout:3973: done (2 left) 2024-09-13T14:32:40.181 INFO:tasks.rados.rados.0.smithi026.stdout:3975: done (1 left) 2024-09-13T14:32:40.181 INFO:tasks.rados.rados.0.smithi026.stdout:3976: snap_remove snap 373 2024-09-13T14:32:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:40 smithi026 ceph-mon[97022]: pgmap v868: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 263 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:32:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:40 smithi026 ceph-mon[99461]: pgmap v868: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 263 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:32:40.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:40 smithi145 ceph-mon[89145]: pgmap v868: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 263 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-13T14:32:40.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: finishing write tid 1 to smithi02683976-25 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: finishing write tid 2 to smithi02683976-25 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: finishing write tid 3 to smithi02683976-25 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: finishing write tid 4 to smithi02683976-25 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 25 v 2033 (ObjNum 1235 snap 380 seq_num 1235) dirty exists 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: left oid 25 (ObjNum 1235 snap 380 seq_num 1235) 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3974: done (1 left) 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3976: done (0 left) 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:3977: rollback oid 10 current snap is 380 2024-09-13T14:32:41.127 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 10 to 362 2024-09-13T14:32:41.128 INFO:tasks.rados.rados.0.smithi026.stdout:3978: setattr oid 18 current snap is 380 2024-09-13T14:32:41.129 INFO:tasks.rados.rados.0.smithi026.stdout:3979: setattr oid 35 current snap is 380 2024-09-13T14:32:41.132 INFO:tasks.rados.rados.0.smithi026.stdout:3977: finishing rollback tid 1 to smithi02683976-10 2024-09-13T14:32:41.132 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 10 v 0 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-13T14:32:41.133 INFO:tasks.rados.rados.0.smithi026.stdout:3977: done (2 left) 2024-09-13T14:32:41.133 INFO:tasks.rados.rados.0.smithi026.stdout:3980: snap_create 2024-09-13T14:32:41.143 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 18 v 2127 (ObjNum 1216 snap 372 seq_num 1216) dirty exists 2024-09-13T14:32:41.146 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 35 v 2803 (ObjNum 305 snap 0 seq_num 519944576) dirty exists 2024-09-13T14:32:41.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:41 smithi026 ceph-mon[97022]: osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:32:41.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:41 smithi026 ceph-mon[99461]: osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:32:41.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:41] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.51.0" 2024-09-13T14:32:41.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:41 smithi145 ceph-mon[89145]: osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:32:42.132 INFO:tasks.rados.rados.0.smithi026.stdout:3978: done (2 left) 2024-09-13T14:32:42.133 INFO:tasks.rados.rados.0.smithi026.stdout:3979: done (1 left) 2024-09-13T14:32:42.133 INFO:tasks.rados.rados.0.smithi026.stdout:3980: done (0 left) 2024-09-13T14:32:42.133 INFO:tasks.rados.rados.0.smithi026.stdout:3981: snap_create 2024-09-13T14:32:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[97022]: pgmap v871: 121 pgs: 121 active+clean; 221 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[97022]: osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:32:42.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[99461]: pgmap v871: 121 pgs: 121 active+clean; 221 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[99461]: osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:32:42.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:42 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:42 smithi145 ceph-mon[89145]: pgmap v871: 121 pgs: 121 active+clean; 221 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:32:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:42 smithi145 ceph-mon[89145]: osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:32:42.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:42 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:43.138 INFO:tasks.rados.rados.0.smithi026.stdout:3981: done (0 left) 2024-09-13T14:32:43.138 INFO:tasks.rados.rados.0.smithi026.stdout:3982: read oid 39 snap 374 2024-09-13T14:32:43.138 INFO:tasks.rados.rados.0.smithi026.stdout:3982: expect (ObjNum 1196 snap 366 seq_num 1196) 2024-09-13T14:32:43.139 INFO:tasks.rados.rados.0.smithi026.stdout:3983: snap_remove snap 379 2024-09-13T14:32:43.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:43 smithi026 ceph-mon[99461]: osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:32:43.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:43 smithi026 ceph-mon[97022]: osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:32:43.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:43 smithi145 ceph-mon[89145]: osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:32:44.057 INFO:tasks.rados.rados.0.smithi026.stdout:3982: expect (ObjNum 1196 snap 366 seq_num 1196) 2024-09-13T14:32:44.234 INFO:tasks.rados.rados.0.smithi026.stdout:3982: done (1 left) 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3983: done (0 left) 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3984: read oid 14 snap -1 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3984: expect (ObjNum 1223 snap 374 seq_num 1223) 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3985: rollback oid 49 current snap is 382 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 49 to 376 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3986: rmattr oid 4 current snap is 382 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3986: done (2 left) 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3987: copy_from oid 11 from oid 5 current snap is 382 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3988: rollback oid 37 current snap is 382 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 37 to 380 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3989: read oid 33 snap -1 2024-09-13T14:32:44.235 INFO:tasks.rados.rados.0.smithi026.stdout:3989: expect (ObjNum 1190 snap 365 seq_num 1190) 2024-09-13T14:32:44.236 INFO:tasks.rados.rados.0.smithi026.stdout:3990: snap_create 2024-09-13T14:32:44.237 INFO:tasks.rados.rados.0.smithi026.stdout:3987: finishing copy_from to smithi02683976-11 2024-09-13T14:32:44.237 INFO:tasks.rados.rados.0.smithi026.stdout:3987: got expected ENOENT (src dne) 2024-09-13T14:32:44.237 INFO:tasks.rados.rados.0.smithi026.stdout:3988: finishing rollback tid 1 to smithi02683976-37 2024-09-13T14:32:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 37 v 1493 (ObjNum 993 snap 0 seq_num 519915616) dirty dne 2024-09-13T14:32:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:3987: finishing copy_from racing read to smithi02683976-11 2024-09-13T14:32:44.238 INFO:tasks.rados.rados.0.smithi026.stdout:3989: expect (ObjNum 1190 snap 365 seq_num 1190) 2024-09-13T14:32:44.267 INFO:tasks.rados.rados.0.smithi026.stdout:3985: finishing rollback tid 0 to smithi02683976-49 2024-09-13T14:32:44.268 INFO:tasks.rados.rados.0.smithi026.stdout:3985: finishing rollback tid 1 to smithi02683976-49 2024-09-13T14:32:44.268 INFO:tasks.rados.rados.0.smithi026.stdout:3985: finishing rollback tid 2 to smithi02683976-49 2024-09-13T14:32:44.268 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 49 v 2057 (ObjNum 1225 snap 374 seq_num 1225) dirty exists 2024-09-13T14:32:44.268 INFO:tasks.rados.rados.0.smithi026.stdout:3984: expect (ObjNum 1223 snap 374 seq_num 1223) 2024-09-13T14:32:44.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[99461]: pgmap v874: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:32:44.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[99461]: osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:32:44.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[99461]: osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:32:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[97022]: pgmap v874: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:32:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[97022]: osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:32:44.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:44 smithi026 ceph-mon[97022]: osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:32:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:44 smithi145 ceph-mon[89145]: pgmap v874: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-13T14:32:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:44 smithi145 ceph-mon[89145]: osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:32:44.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:44 smithi145 ceph-mon[89145]: osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3984: done (5 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3985: done (4 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3987: done (3 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3988: done (2 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3989: done (1 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3990: done (0 left) 2024-09-13T14:32:45.144 INFO:tasks.rados.rados.0.smithi026.stdout:3991: delete oid 12 current snap is 383 2024-09-13T14:32:45.151 INFO:tasks.rados.rados.0.smithi026.stdout:3991: done (0 left) 2024-09-13T14:32:45.151 INFO:tasks.rados.rados.0.smithi026.stdout:3992: snap_create 2024-09-13T14:32:46.145 INFO:tasks.rados.rados.0.smithi026.stdout:3992: done (0 left) 2024-09-13T14:32:46.146 INFO:tasks.rados.rados.0.smithi026.stdout:3993: delete oid 41 current snap is 384 2024-09-13T14:32:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3993: done (0 left) 2024-09-13T14:32:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3994: read oid 22 snap 377 2024-09-13T14:32:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3994: expect deleted 2024-09-13T14:32:46.150 INFO:tasks.rados.rados.0.smithi026.stdout:3995: snap_create 2024-09-13T14:32:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:46 smithi026 ceph-mon[97022]: pgmap v877: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:32:46.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:46 smithi026 ceph-mon[97022]: osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:32:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:46 smithi026 ceph-mon[99461]: pgmap v877: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:32:46.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:46 smithi026 ceph-mon[99461]: osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:32:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:46 smithi145 ceph-mon[89145]: pgmap v877: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 7 op/s 2024-09-13T14:32:46.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:46 smithi145 ceph-mon[89145]: osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:32:47.158 INFO:tasks.rados.rados.0.smithi026.stdout:3994: done (1 left) 2024-09-13T14:32:47.158 INFO:tasks.rados.rados.0.smithi026.stdout:3995: done (0 left) 2024-09-13T14:32:47.158 INFO:tasks.rados.rados.0.smithi026.stdout:3996: read oid 42 snap -1 2024-09-13T14:32:47.158 INFO:tasks.rados.rados.0.smithi026.stdout:3996: expect deleted 2024-09-13T14:32:47.158 INFO:tasks.rados.rados.0.smithi026.stdout:3997: snap_remove snap 374 2024-09-13T14:32:47.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:47 smithi026 ceph-mon[97022]: osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:32:47.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:47 smithi026 ceph-mon[99461]: osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:32:47.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:47 smithi145 ceph-mon[89145]: osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:32:48.167 INFO:tasks.rados.rados.0.smithi026.stdout:3996: done (1 left) 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout:3997: done (0 left) 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout:3998: read oid 12 snap -1 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout:3998: expect deleted 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout:3999: rollback oid 23 current snap is 385 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout:rollback oid 23 to 377 2024-09-13T14:32:48.168 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 2 2024-09-13T14:32:48.169 INFO:tasks.rados.rados.0.smithi026.stdout:3998: done (1 left) 2024-09-13T14:32:48.169 INFO:tasks.rados.rados.0.smithi026.stdout: waiting on 1 2024-09-13T14:32:48.184 INFO:tasks.rados.rados.0.smithi026.stdout:3999: finishing rollback tid 0 to smithi02683976-23 2024-09-13T14:32:48.184 INFO:tasks.rados.rados.0.smithi026.stdout:3999: finishing rollback tid 1 to smithi02683976-23 2024-09-13T14:32:48.184 INFO:tasks.rados.rados.0.smithi026.stdout:3999: finishing rollback tid 2 to smithi02683976-23 2024-09-13T14:32:48.184 INFO:tasks.rados.rados.0.smithi026.stdout:update_object_version oid 23 v 2137 (ObjNum 917 snap 286 seq_num 917) dirty exists 2024-09-13T14:32:48.185 INFO:tasks.rados.rados.0.smithi026.stdout:3999: done (0 left) 2024-09-13T14:32:48.188 INFO:tasks.rados.rados.0.smithi026.stderr:0 errors. 2024-09-13T14:32:48.188 INFO:tasks.rados.rados.0.smithi026.stderr: 2024-09-13T14:32:48.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:32:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:48 smithi026 ceph-mon[99461]: pgmap v880: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:32:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:48 smithi026 ceph-mon[99461]: osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:32:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:48 smithi026 ceph-mon[97022]: pgmap v880: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:32:48.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:48 smithi026 ceph-mon[97022]: osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:32:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:48 smithi145 ceph-mon[89145]: pgmap v880: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:32:48.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:48 smithi145 ceph-mon[89145]: osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:32:48.489 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:49.404 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:49 smithi026 ceph-mon[99461]: osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:32:49.404 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:49 smithi026 ceph-mon[99461]: osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:32:49.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:49 smithi026 ceph-mon[97022]: osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:32:49.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:49 smithi026 ceph-mon[97022]: osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:32:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:49 smithi145 ceph-mon[89145]: osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:32:49.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:49 smithi145 ceph-mon[89145]: osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:32:50.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:50 smithi026 ceph-mon[99461]: pgmap v883: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:32:50.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:50 smithi026 ceph-mon[97022]: pgmap v883: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:32:50.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:50 smithi145 ceph-mon[89145]: pgmap v883: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 0 B/s wr, 13 op/s 2024-09-13T14:32:50.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:32:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:32:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:50.610 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:32:50.610 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":849,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:32:49.052432+0000","last_up_change":"2024-09-13T14:27:54.980682+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-13T14:16:19.892063+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-13T14:19:17.928621+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"848","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":481,"snap_epoch":848,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":332,"up_from":346,"up_thru":576,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6803","nonce":3754976775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6805","nonce":3754976775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6809","nonce":3754976775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6807","nonce":3754976775}]},"public_addr":"172.21.15.26:6803/3754976775","cluster_addr":"172.21.15.26:6805/3754976775","heartbeat_back_addr":"172.21.15.26:6809/3754976775","heartbeat_front_addr":"172.21.15.26:6807/3754976775","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":366,"up_from":378,"up_thru":576,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6811","nonce":2254834333}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6813","nonce":2254834333}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6817","nonce":2254834333}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6815","nonce":2254834333}]},"public_addr":"172.21.15.26:6811/2254834333","cluster_addr":"172.21.15.26:6813/2254834333","heartbeat_back_addr":"172.21.15.26:6817/2254834333","heartbeat_front_addr":"172.21.15.26:6815/2254834333","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":399,"up_from":412,"up_thru":576,"down_at":400,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6819","nonce":718889581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6821","nonce":718889581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6825","nonce":718889581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6823","nonce":718889581}]},"public_addr":"172.21.15.26:6819/718889581","cluster_addr":"172.21.15.26:6821/718889581","heartbeat_back_addr":"172.21.15.26:6825/718889581","heartbeat_front_addr":"172.21.15.26:6823/718889581","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":430,"up_from":443,"up_thru":576,"down_at":431,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6827","nonce":3139338605}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6829","nonce":3139338605}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6833","nonce":3139338605}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6831","nonce":3139338605}]},"public_addr":"172.21.15.26:6827/3139338605","cluster_addr":"172.21.15.26:6829/3139338605","heartbeat_back_addr":"172.21.15.26:6833/3139338605","heartbeat_front_addr":"172.21.15.26:6831/3139338605","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":463,"up_from":474,"up_thru":576,"down_at":464,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6801","nonce":3668564396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6803","nonce":3668564396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6807","nonce":3668564396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6805","nonce":3668564396}]},"public_addr":"172.21.15.145:6801/3668564396","cluster_addr":"172.21.15.145:6803/3668564396","heartbeat_back_addr":"172.21.15.145:6807/3668564396","heartbeat_front_addr":"172.21.15.145:6805/3668564396","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":495,"up_from":508,"up_thru":576,"down_at":496,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6809","nonce":313394676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6811","nonce":313394676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6815","nonce":313394676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6813","nonce":313394676}]},"public_addr":"172.21.15.145:6809/313394676","cluster_addr":"172.21.15.145:6811/313394676","heartbeat_back_addr":"172.21.15.145:6815/313394676","heartbeat_front_addr":"172.21.15.145:6813/313394676","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":529,"up_from":543,"up_thru":576,"down_at":530,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6817","nonce":3319036697}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6819","nonce":3319036697}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6823","nonce":3319036697}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6821","nonce":3319036697}]},"public_addr":"172.21.15.145:6817/3319036697","cluster_addr":"172.21.15.145:6819/3319036697","heartbeat_back_addr":"172.21.15.145:6823/3319036697","heartbeat_front_addr":"172.21.15.145:6821/3319036697","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":563,"up_from":576,"up_thru":576,"down_at":564,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6825","nonce":2839525764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6827","nonce":2839525764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6831","nonce":2839525764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6829","nonce":2839525764}]},"public_addr":"172.21.15.145:6825/2839525764","cluster_addr":"172.21.15.145:6827/2839525764","heartbeat_back_addr":"172.21.15.145:6831/2839525764","heartbeat_front_addr":"172.21.15.145:6829/2839525764","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":332},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":366},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":398},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":430},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":463},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":495},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":529},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":563}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6833/2983197144":"2024-09-14T14:28:17.341368+0000","172.21.15.145:6832/2983197144":"2024-09-14T14:28:17.341368+0000","172.21.15.26:0/1320392898":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/3486505461":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/3971773587":"2024-09-14T14:22:40.678958+0000","172.21.15.26:6801/2523901053":"2024-09-14T14:22:40.678958+0000","172.21.15.26:6800/2523901053":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/2708858480":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/602611566":"2024-09-14T14:22:40.678958+0000","172.21.15.145:6835/1781482491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/2247322222":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.145:0/1853142526":"2024-09-14T14:21:59.347678+0000","172.21.15.145:0/203034452":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.145:0/1500293491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2245841676":"2024-09-14T14:20:46.093511+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6801/295355582":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4132567630":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/1842776456":"2024-09-14T14:16:17.348500+0000","172.21.15.145:0/2745661972":"2024-09-14T14:21:59.347678+0000","172.21.15.145:0/3573890460":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2209085404":"2024-09-14T14:20:46.093511+0000","172.21.15.145:6834/1781482491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3442380687":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/173360804":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/2134193589":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/2881487463":"2024-09-14T14:20:46.093511+0000","172.21.15.26:6800/295355582":"2024-09-14T14:20:46.093511+0000","172.21.15.145:0/93878221":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2881792973":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6801/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/2605336872":"2024-09-14T14:22:40.678958+0000","172.21.15.145:0/2960544754":"2024-09-14T14:21:59.347678+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":471,"length":1},{"begin":476,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":470,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:32:51.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:32:51.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:51 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/938062188' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:32:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:51 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/938062188' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:32:51.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:51 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/938062188' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:32:51.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:32:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:32:51] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.51.0" 2024-09-13T14:32:52.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:52 smithi026 ceph-mon[99461]: pgmap v885: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:52.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:52 smithi026 ceph-mon[97022]: pgmap v885: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:52.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:52 smithi145 ceph-mon[89145]: pgmap v885: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:53.473 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:32:53.474 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-09-13T14:32:54.073 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":886,"stamp":"2024-09-13T14:32:52.849856+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":233266173,"num_objects":128,"num_object_clones":57,"num_object_copies":384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":128,"num_whiteouts":9,"num_read":4702,"num_read_kb":6197170,"num_write":8706,"num_write_kb":1957209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":437740064,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19790,"ondisk_log_size":19790,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2874100,"kb_used_data":432756,"kb_used_omap":242,"kb_used_meta":2440973,"kb_avail":747087116,"statfs":{"total":767960285184,"available":765017206784,"internally_reserved":0,"allocated":443142144,"data_stored":638120507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":247998,"internal_metadata":2499557186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-3866624,"num_objects":-4,"num_object_clones":-4,"num_object_copies":-12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4,"num_whiteouts":1,"num_read":5,"num_read_kb":9404,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001480"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":1069,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165846+0000","last_change":"2024-09-13T14:27:56.006611+0000","last_active":"2024-09-13T14:32:48.165846+0000","last_peered":"2024-09-13T14:32:48.165846+0000","last_clean":"2024-09-13T14:32:48.165846+0000","last_became_active":"2024-09-13T14:27:56.003268+0000","last_became_peered":"2024-09-13T14:27:56.003268+0000","last_unstale":"2024-09-13T14:32:48.165846+0000","last_undegraded":"2024-09-13T14:32:48.165846+0000","last_fullsized":"2024-09-13T14:32:48.165846+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:17.267760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":1274,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411614+0000","last_change":"2024-09-13T14:27:22.030895+0000","last_active":"2024-09-13T14:32:49.411614+0000","last_peered":"2024-09-13T14:32:49.411614+0000","last_clean":"2024-09-13T14:32:49.411614+0000","last_became_active":"2024-09-13T14:27:22.029872+0000","last_became_peered":"2024-09-13T14:27:22.029872+0000","last_unstale":"2024-09-13T14:32:49.411614+0000","last_undegraded":"2024-09-13T14:32:49.411614+0000","last_fullsized":"2024-09-13T14:32:49.411614+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:36.755507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":1286,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130271+0000","last_change":"2024-09-13T14:25:07.222271+0000","last_active":"2024-09-13T14:32:49.130271+0000","last_peered":"2024-09-13T14:32:49.130271+0000","last_clean":"2024-09-13T14:32:49.130271+0000","last_became_active":"2024-09-13T14:25:07.221872+0000","last_became_peered":"2024-09-13T14:25:07.221872+0000","last_unstale":"2024-09-13T14:32:49.130271+0000","last_undegraded":"2024-09-13T14:32:49.130271+0000","last_fullsized":"2024-09-13T14:32:49.130271+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:30.138850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":1151,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411345+0000","last_change":"2024-09-13T14:27:56.005122+0000","last_active":"2024-09-13T14:32:49.411345+0000","last_peered":"2024-09-13T14:32:49.411345+0000","last_clean":"2024-09-13T14:32:49.411345+0000","last_became_active":"2024-09-13T14:27:56.004520+0000","last_became_peered":"2024-09-13T14:27:56.004520+0000","last_unstale":"2024-09-13T14:32:49.411345+0000","last_undegraded":"2024-09-13T14:32:49.411345+0000","last_fullsized":"2024-09-13T14:32:49.411345+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:42:41.284079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":1082,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165199+0000","last_change":"2024-09-13T14:27:56.007564+0000","last_active":"2024-09-13T14:32:48.165199+0000","last_peered":"2024-09-13T14:32:48.165199+0000","last_clean":"2024-09-13T14:32:48.165199+0000","last_became_active":"2024-09-13T14:27:56.006782+0000","last_became_peered":"2024-09-13T14:27:56.006782+0000","last_unstale":"2024-09-13T14:32:48.165199+0000","last_undegraded":"2024-09-13T14:32:48.165199+0000","last_fullsized":"2024-09-13T14:32:48.165199+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:22.521888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":1132,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411664+0000","last_change":"2024-09-13T14:26:46.974311+0000","last_active":"2024-09-13T14:32:49.411664+0000","last_peered":"2024-09-13T14:32:49.411664+0000","last_clean":"2024-09-13T14:32:49.411664+0000","last_became_active":"2024-09-13T14:26:46.973927+0000","last_became_peered":"2024-09-13T14:26:46.973927+0000","last_unstale":"2024-09-13T14:32:49.411664+0000","last_undegraded":"2024-09-13T14:32:49.411664+0000","last_fullsized":"2024-09-13T14:32:49.411664+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:16:51.625127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":1082,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164275+0000","last_change":"2024-09-13T14:27:56.004763+0000","last_active":"2024-09-13T14:32:48.164275+0000","last_peered":"2024-09-13T14:32:48.164275+0000","last_clean":"2024-09-13T14:32:48.164275+0000","last_became_active":"2024-09-13T14:27:56.004329+0000","last_became_peered":"2024-09-13T14:27:56.004329+0000","last_unstale":"2024-09-13T14:32:48.164275+0000","last_undegraded":"2024-09-13T14:32:48.164275+0000","last_fullsized":"2024-09-13T14:32:48.164275+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:28.079994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":1206,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.144630+0000","last_change":"2024-09-13T14:27:22.032479+0000","last_active":"2024-09-13T14:32:46.144630+0000","last_peered":"2024-09-13T14:32:46.144630+0000","last_clean":"2024-09-13T14:32:46.144630+0000","last_became_active":"2024-09-13T14:27:22.032045+0000","last_became_peered":"2024-09-13T14:27:22.032045+0000","last_unstale":"2024-09-13T14:32:46.144630+0000","last_undegraded":"2024-09-13T14:32:46.144630+0000","last_fullsized":"2024-09-13T14:32:46.144630+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.386712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":1076,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165621+0000","last_change":"2024-09-13T14:27:56.010938+0000","last_active":"2024-09-13T14:32:48.165621+0000","last_peered":"2024-09-13T14:32:48.165621+0000","last_clean":"2024-09-13T14:32:48.165621+0000","last_became_active":"2024-09-13T14:27:56.010619+0000","last_became_peered":"2024-09-13T14:27:56.010619+0000","last_unstale":"2024-09-13T14:32:48.165621+0000","last_undegraded":"2024-09-13T14:32:48.165621+0000","last_fullsized":"2024-09-13T14:32:48.165621+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:36.626954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":1107,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412260+0000","last_change":"2024-09-13T14:27:22.030019+0000","last_active":"2024-09-13T14:32:49.412260+0000","last_peered":"2024-09-13T14:32:49.412260+0000","last_clean":"2024-09-13T14:32:49.412260+0000","last_became_active":"2024-09-13T14:27:22.029688+0000","last_became_peered":"2024-09-13T14:27:22.029688+0000","last_unstale":"2024-09-13T14:32:49.412260+0000","last_undegraded":"2024-09-13T14:32:49.412260+0000","last_fullsized":"2024-09-13T14:32:49.412260+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:33:30.421284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":1107,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412034+0000","last_change":"2024-09-13T14:27:22.028123+0000","last_active":"2024-09-13T14:32:49.412034+0000","last_peered":"2024-09-13T14:32:49.412034+0000","last_clean":"2024-09-13T14:32:49.412034+0000","last_became_active":"2024-09-13T14:27:22.027778+0000","last_became_peered":"2024-09-13T14:27:22.027778+0000","last_unstale":"2024-09-13T14:32:49.412034+0000","last_undegraded":"2024-09-13T14:32:49.412034+0000","last_fullsized":"2024-09-13T14:32:49.412034+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:00.545081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":1172,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056864+0000","last_change":"2024-09-13T14:27:22.022019+0000","last_active":"2024-09-13T14:32:49.056864+0000","last_peered":"2024-09-13T14:32:49.056864+0000","last_clean":"2024-09-13T14:32:49.056864+0000","last_became_active":"2024-09-13T14:27:22.021714+0000","last_became_peered":"2024-09-13T14:27:22.021714+0000","last_unstale":"2024-09-13T14:32:49.056864+0000","last_undegraded":"2024-09-13T14:32:49.056864+0000","last_fullsized":"2024-09-13T14:32:49.056864+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:05.620942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":1208,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.144390+0000","last_change":"2024-09-13T14:27:22.028316+0000","last_active":"2024-09-13T14:32:46.144390+0000","last_peered":"2024-09-13T14:32:46.144390+0000","last_clean":"2024-09-13T14:32:46.144390+0000","last_became_active":"2024-09-13T14:27:22.027940+0000","last_became_peered":"2024-09-13T14:27:22.027940+0000","last_unstale":"2024-09-13T14:32:46.144390+0000","last_undegraded":"2024-09-13T14:32:46.144390+0000","last_fullsized":"2024-09-13T14:32:46.144390+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:57.363621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":1285,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.129936+0000","last_change":"2024-09-13T14:26:46.970568+0000","last_active":"2024-09-13T14:32:49.129936+0000","last_peered":"2024-09-13T14:32:49.129936+0000","last_clean":"2024-09-13T14:32:49.129936+0000","last_became_active":"2024-09-13T14:26:46.970219+0000","last_became_peered":"2024-09-13T14:26:46.970219+0000","last_unstale":"2024-09-13T14:32:49.129936+0000","last_undegraded":"2024-09-13T14:32:49.129936+0000","last_fullsized":"2024-09-13T14:32:49.129936+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:27.845175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":1264,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411099+0000","last_change":"2024-09-13T14:26:46.968334+0000","last_active":"2024-09-13T14:32:49.411099+0000","last_peered":"2024-09-13T14:32:49.411099+0000","last_clean":"2024-09-13T14:32:49.411099+0000","last_became_active":"2024-09-13T14:26:46.967997+0000","last_became_peered":"2024-09-13T14:26:46.967997+0000","last_unstale":"2024-09-13T14:32:49.411099+0000","last_undegraded":"2024-09-13T14:32:49.411099+0000","last_fullsized":"2024-09-13T14:32:49.411099+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:28.075168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":1136,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411062+0000","last_change":"2024-09-13T14:27:56.003398+0000","last_active":"2024-09-13T14:32:49.411062+0000","last_peered":"2024-09-13T14:32:49.411062+0000","last_clean":"2024-09-13T14:32:49.411062+0000","last_became_active":"2024-09-13T14:27:56.003082+0000","last_became_peered":"2024-09-13T14:27:56.003082+0000","last_unstale":"2024-09-13T14:32:49.411062+0000","last_undegraded":"2024-09-13T14:32:49.411062+0000","last_fullsized":"2024-09-13T14:32:49.411062+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:04:00.049200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":1139,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412496+0000","last_change":"2024-09-13T14:27:56.004558+0000","last_active":"2024-09-13T14:32:49.412496+0000","last_peered":"2024-09-13T14:32:49.412496+0000","last_clean":"2024-09-13T14:32:49.412496+0000","last_became_active":"2024-09-13T14:27:56.004258+0000","last_became_peered":"2024-09-13T14:27:56.004258+0000","last_unstale":"2024-09-13T14:32:49.412496+0000","last_undegraded":"2024-09-13T14:32:49.412496+0000","last_fullsized":"2024-09-13T14:32:49.412496+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:59:32.081194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":1238,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055714+0000","last_change":"2024-09-13T14:27:56.005923+0000","last_active":"2024-09-13T14:32:49.055714+0000","last_peered":"2024-09-13T14:32:49.055714+0000","last_clean":"2024-09-13T14:32:49.055714+0000","last_became_active":"2024-09-13T14:27:56.005582+0000","last_became_peered":"2024-09-13T14:27:56.005582+0000","last_unstale":"2024-09-13T14:32:49.055714+0000","last_undegraded":"2024-09-13T14:32:49.055714+0000","last_fullsized":"2024-09-13T14:32:49.055714+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:29.924781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":1069,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165361+0000","last_change":"2024-09-13T14:27:56.012700+0000","last_active":"2024-09-13T14:32:48.165361+0000","last_peered":"2024-09-13T14:32:48.165361+0000","last_clean":"2024-09-13T14:32:48.165361+0000","last_became_active":"2024-09-13T14:27:56.011726+0000","last_became_peered":"2024-09-13T14:27:56.011726+0000","last_unstale":"2024-09-13T14:32:48.165361+0000","last_undegraded":"2024-09-13T14:32:48.165361+0000","last_fullsized":"2024-09-13T14:32:48.165361+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:33.166363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":1115,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411619+0000","last_change":"2024-09-13T14:27:22.029858+0000","last_active":"2024-09-13T14:32:49.411619+0000","last_peered":"2024-09-13T14:32:49.411619+0000","last_clean":"2024-09-13T14:32:49.411619+0000","last_became_active":"2024-09-13T14:27:22.029550+0000","last_became_peered":"2024-09-13T14:27:22.029550+0000","last_unstale":"2024-09-13T14:32:49.411619+0000","last_undegraded":"2024-09-13T14:32:49.411619+0000","last_fullsized":"2024-09-13T14:32:49.411619+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:58:55.209822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":1285,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.129980+0000","last_change":"2024-09-13T14:27:56.003968+0000","last_active":"2024-09-13T14:32:49.129980+0000","last_peered":"2024-09-13T14:32:49.129980+0000","last_clean":"2024-09-13T14:32:49.129980+0000","last_became_active":"2024-09-13T14:27:56.003591+0000","last_became_peered":"2024-09-13T14:27:56.003591+0000","last_unstale":"2024-09-13T14:32:49.129980+0000","last_undegraded":"2024-09-13T14:32:49.129980+0000","last_fullsized":"2024-09-13T14:32:49.129980+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:25.366286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":1125,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411272+0000","last_change":"2024-09-13T14:27:56.000030+0000","last_active":"2024-09-13T14:32:49.411272+0000","last_peered":"2024-09-13T14:32:49.411272+0000","last_clean":"2024-09-13T14:32:49.411272+0000","last_became_active":"2024-09-13T14:27:55.999746+0000","last_became_peered":"2024-09-13T14:27:55.999746+0000","last_unstale":"2024-09-13T14:32:49.411272+0000","last_undegraded":"2024-09-13T14:32:49.411272+0000","last_fullsized":"2024-09-13T14:32:49.411272+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:46.414114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":1209,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.145002+0000","last_change":"2024-09-13T14:26:12.510216+0000","last_active":"2024-09-13T14:32:46.145002+0000","last_peered":"2024-09-13T14:32:46.145002+0000","last_clean":"2024-09-13T14:32:46.145002+0000","last_became_active":"2024-09-13T14:26:12.509853+0000","last_became_peered":"2024-09-13T14:26:12.509853+0000","last_unstale":"2024-09-13T14:32:46.145002+0000","last_undegraded":"2024-09-13T14:32:46.145002+0000","last_fullsized":"2024-09-13T14:32:46.145002+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:35.953969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":1156,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411293+0000","last_change":"2024-09-13T14:26:46.970184+0000","last_active":"2024-09-13T14:32:49.411293+0000","last_peered":"2024-09-13T14:32:49.411293+0000","last_clean":"2024-09-13T14:32:49.411293+0000","last_became_active":"2024-09-13T14:26:46.969855+0000","last_became_peered":"2024-09-13T14:26:46.969855+0000","last_unstale":"2024-09-13T14:32:49.411293+0000","last_undegraded":"2024-09-13T14:32:49.411293+0000","last_fullsized":"2024-09-13T14:32:49.411293+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:02.707672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":1201,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143244+0000","last_change":"2024-09-13T14:26:12.508055+0000","last_active":"2024-09-13T14:32:46.143244+0000","last_peered":"2024-09-13T14:32:46.143244+0000","last_clean":"2024-09-13T14:32:46.143244+0000","last_became_active":"2024-09-13T14:26:12.507629+0000","last_became_peered":"2024-09-13T14:26:12.507629+0000","last_unstale":"2024-09-13T14:32:46.143244+0000","last_undegraded":"2024-09-13T14:32:46.143244+0000","last_fullsized":"2024-09-13T14:32:46.143244+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:26:01.023922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":1109,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411824+0000","last_change":"2024-09-13T14:27:56.003320+0000","last_active":"2024-09-13T14:32:49.411824+0000","last_peered":"2024-09-13T14:32:49.411824+0000","last_clean":"2024-09-13T14:32:49.411824+0000","last_became_active":"2024-09-13T14:27:56.003030+0000","last_became_peered":"2024-09-13T14:27:56.003030+0000","last_unstale":"2024-09-13T14:32:49.411824+0000","last_undegraded":"2024-09-13T14:32:49.411824+0000","last_fullsized":"2024-09-13T14:32:49.411824+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:44.661948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":1264,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411505+0000","last_change":"2024-09-13T14:25:39.987575+0000","last_active":"2024-09-13T14:32:49.411505+0000","last_peered":"2024-09-13T14:32:49.411505+0000","last_clean":"2024-09-13T14:32:49.411505+0000","last_became_active":"2024-09-13T14:25:39.987149+0000","last_became_peered":"2024-09-13T14:25:39.987149+0000","last_unstale":"2024-09-13T14:32:49.411505+0000","last_undegraded":"2024-09-13T14:32:49.411505+0000","last_fullsized":"2024-09-13T14:32:49.411505+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:29.346650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":1218,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143271+0000","last_change":"2024-09-13T14:26:12.508843+0000","last_active":"2024-09-13T14:32:46.143271+0000","last_peered":"2024-09-13T14:32:46.143271+0000","last_clean":"2024-09-13T14:32:46.143271+0000","last_became_active":"2024-09-13T14:26:12.507651+0000","last_became_peered":"2024-09-13T14:26:12.507651+0000","last_unstale":"2024-09-13T14:32:46.143271+0000","last_undegraded":"2024-09-13T14:32:46.143271+0000","last_fullsized":"2024-09-13T14:32:46.143271+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:45.512668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":1286,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130166+0000","last_change":"2024-09-13T14:26:12.507055+0000","last_active":"2024-09-13T14:32:49.130166+0000","last_peered":"2024-09-13T14:32:49.130166+0000","last_clean":"2024-09-13T14:32:49.130166+0000","last_became_active":"2024-09-13T14:26:12.506516+0000","last_became_peered":"2024-09-13T14:26:12.506516+0000","last_unstale":"2024-09-13T14:32:49.130166+0000","last_undegraded":"2024-09-13T14:32:49.130166+0000","last_fullsized":"2024-09-13T14:32:49.130166+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:43.316737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":1156,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411389+0000","last_change":"2024-09-13T14:26:46.970516+0000","last_active":"2024-09-13T14:32:49.411389+0000","last_peered":"2024-09-13T14:32:49.411389+0000","last_clean":"2024-09-13T14:32:49.411389+0000","last_became_active":"2024-09-13T14:26:46.969612+0000","last_became_peered":"2024-09-13T14:26:46.969612+0000","last_unstale":"2024-09-13T14:32:49.411389+0000","last_undegraded":"2024-09-13T14:32:49.411389+0000","last_fullsized":"2024-09-13T14:32:49.411389+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:54.579332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":1238,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055330+0000","last_change":"2024-09-13T14:27:22.021236+0000","last_active":"2024-09-13T14:32:49.055330+0000","last_peered":"2024-09-13T14:32:49.055330+0000","last_clean":"2024-09-13T14:32:49.055330+0000","last_became_active":"2024-09-13T14:27:22.020884+0000","last_became_peered":"2024-09-13T14:27:22.020884+0000","last_unstale":"2024-09-13T14:32:49.055330+0000","last_undegraded":"2024-09-13T14:32:49.055330+0000","last_fullsized":"2024-09-13T14:32:49.055330+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:04.601063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":1107,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411034+0000","last_change":"2024-09-13T14:27:22.022721+0000","last_active":"2024-09-13T14:32:49.411034+0000","last_peered":"2024-09-13T14:32:49.411034+0000","last_clean":"2024-09-13T14:32:49.411034+0000","last_became_active":"2024-09-13T14:27:22.019121+0000","last_became_peered":"2024-09-13T14:27:22.019121+0000","last_unstale":"2024-09-13T14:32:49.411034+0000","last_undegraded":"2024-09-13T14:32:49.411034+0000","last_fullsized":"2024-09-13T14:32:49.411034+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:07.574220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":1203,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143025+0000","last_change":"2024-09-13T14:25:39.987571+0000","last_active":"2024-09-13T14:32:46.143025+0000","last_peered":"2024-09-13T14:32:46.143025+0000","last_clean":"2024-09-13T14:32:46.143025+0000","last_became_active":"2024-09-13T14:25:39.984323+0000","last_became_peered":"2024-09-13T14:25:39.984323+0000","last_unstale":"2024-09-13T14:32:46.143025+0000","last_undegraded":"2024-09-13T14:32:46.143025+0000","last_fullsized":"2024-09-13T14:32:46.143025+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:54.249798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1b","version":"598'2","reported_seq":1213,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.144504+0000","last_change":"2024-09-13T14:27:56.003386+0000","last_active":"2024-09-13T14:32:46.144504+0000","last_peered":"2024-09-13T14:32:46.144504+0000","last_clean":"2024-09-13T14:32:46.144504+0000","last_became_active":"2024-09-13T14:27:56.003018+0000","last_became_peered":"2024-09-13T14:27:56.003018+0000","last_unstale":"2024-09-13T14:32:46.144504+0000","last_undegraded":"2024-09-13T14:32:46.144504+0000","last_fullsized":"2024-09-13T14:32:46.144504+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:54.294138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1238,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056601+0000","last_change":"2024-09-13T14:27:56.005931+0000","last_active":"2024-09-13T14:32:49.056601+0000","last_peered":"2024-09-13T14:32:49.056601+0000","last_clean":"2024-09-13T14:32:49.056601+0000","last_became_active":"2024-09-13T14:27:56.005611+0000","last_became_peered":"2024-09-13T14:27:56.005611+0000","last_unstale":"2024-09-13T14:32:49.056601+0000","last_undegraded":"2024-09-13T14:32:49.056601+0000","last_fullsized":"2024-09-13T14:32:49.056601+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:59.316228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"598'2","reported_seq":1114,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411539+0000","last_change":"2024-09-13T14:27:56.000242+0000","last_active":"2024-09-13T14:32:49.411539+0000","last_peered":"2024-09-13T14:32:49.411539+0000","last_clean":"2024-09-13T14:32:49.411539+0000","last_became_active":"2024-09-13T14:27:55.999836+0000","last_became_peered":"2024-09-13T14:27:55.999836+0000","last_unstale":"2024-09-13T14:32:49.411539+0000","last_undegraded":"2024-09-13T14:32:49.411539+0000","last_fullsized":"2024-09-13T14:32:49.411539+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:11:41.764311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":1220,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.144756+0000","last_change":"2024-09-13T14:27:22.021347+0000","last_active":"2024-09-13T14:32:46.144756+0000","last_peered":"2024-09-13T14:32:46.144756+0000","last_clean":"2024-09-13T14:32:46.144756+0000","last_became_active":"2024-09-13T14:27:22.020970+0000","last_became_peered":"2024-09-13T14:27:22.020970+0000","last_unstale":"2024-09-13T14:32:46.144756+0000","last_undegraded":"2024-09-13T14:32:46.144756+0000","last_fullsized":"2024-09-13T14:32:46.144756+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:14:31.913359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1214,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.142936+0000","last_change":"2024-09-13T14:25:39.986237+0000","last_active":"2024-09-13T14:32:46.142936+0000","last_peered":"2024-09-13T14:32:46.142936+0000","last_clean":"2024-09-13T14:32:46.142936+0000","last_became_active":"2024-09-13T14:25:39.984550+0000","last_became_peered":"2024-09-13T14:25:39.984550+0000","last_unstale":"2024-09-13T14:32:46.142936+0000","last_undegraded":"2024-09-13T14:32:46.142936+0000","last_fullsized":"2024-09-13T14:32:46.142936+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:50.609367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":1142,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411188+0000","last_change":"2024-09-13T14:27:56.001496+0000","last_active":"2024-09-13T14:32:49.411188+0000","last_peered":"2024-09-13T14:32:49.411188+0000","last_clean":"2024-09-13T14:32:49.411188+0000","last_became_active":"2024-09-13T14:27:56.000881+0000","last_became_peered":"2024-09-13T14:27:56.000881+0000","last_unstale":"2024-09-13T14:32:49.411188+0000","last_undegraded":"2024-09-13T14:32:49.411188+0000","last_fullsized":"2024-09-13T14:32:49.411188+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:23.599990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1264,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411033+0000","last_change":"2024-09-13T14:26:12.513997+0000","last_active":"2024-09-13T14:32:49.411033+0000","last_peered":"2024-09-13T14:32:49.411033+0000","last_clean":"2024-09-13T14:32:49.411033+0000","last_became_active":"2024-09-13T14:26:12.513265+0000","last_became_peered":"2024-09-13T14:26:12.513265+0000","last_unstale":"2024-09-13T14:32:49.411033+0000","last_undegraded":"2024-09-13T14:32:49.411033+0000","last_fullsized":"2024-09-13T14:32:49.411033+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:41.038242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":1110,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.413699+0000","last_change":"2024-09-13T14:27:22.028088+0000","last_active":"2024-09-13T14:32:49.413699+0000","last_peered":"2024-09-13T14:32:49.413699+0000","last_clean":"2024-09-13T14:32:49.413699+0000","last_became_active":"2024-09-13T14:27:22.027709+0000","last_became_peered":"2024-09-13T14:27:22.027709+0000","last_unstale":"2024-09-13T14:32:49.413699+0000","last_undegraded":"2024-09-13T14:32:49.413699+0000","last_fullsized":"2024-09-13T14:32:49.413699+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:56.320135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1126,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412798+0000","last_change":"2024-09-13T14:27:56.005979+0000","last_active":"2024-09-13T14:32:49.412798+0000","last_peered":"2024-09-13T14:32:49.412798+0000","last_clean":"2024-09-13T14:32:49.412798+0000","last_became_active":"2024-09-13T14:27:56.005151+0000","last_became_peered":"2024-09-13T14:27:56.005151+0000","last_unstale":"2024-09-13T14:32:49.412798+0000","last_undegraded":"2024-09-13T14:32:49.412798+0000","last_fullsized":"2024-09-13T14:32:49.412798+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:55:00.486026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":1162,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412980+0000","last_change":"2024-09-13T14:27:22.025323+0000","last_active":"2024-09-13T14:32:49.412980+0000","last_peered":"2024-09-13T14:32:49.412980+0000","last_clean":"2024-09-13T14:32:49.412980+0000","last_became_active":"2024-09-13T14:27:22.024790+0000","last_became_peered":"2024-09-13T14:27:22.024790+0000","last_unstale":"2024-09-13T14:32:49.412980+0000","last_undegraded":"2024-09-13T14:32:49.412980+0000","last_fullsized":"2024-09-13T14:32:49.412980+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:34.523643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1285,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130329+0000","last_change":"2024-09-13T14:26:46.974062+0000","last_active":"2024-09-13T14:32:49.130329+0000","last_peered":"2024-09-13T14:32:49.130329+0000","last_clean":"2024-09-13T14:32:49.130329+0000","last_became_active":"2024-09-13T14:26:46.973307+0000","last_became_peered":"2024-09-13T14:26:46.973307+0000","last_unstale":"2024-09-13T14:32:49.130329+0000","last_undegraded":"2024-09-13T14:32:49.130329+0000","last_fullsized":"2024-09-13T14:32:49.130329+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:48.818598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":1282,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411568+0000","last_change":"2024-09-13T14:26:46.966365+0000","last_active":"2024-09-13T14:32:49.411568+0000","last_peered":"2024-09-13T14:32:49.411568+0000","last_clean":"2024-09-13T14:32:49.411568+0000","last_became_active":"2024-09-13T14:26:46.965983+0000","last_became_peered":"2024-09-13T14:26:46.965983+0000","last_unstale":"2024-09-13T14:32:49.411568+0000","last_undegraded":"2024-09-13T14:32:49.411568+0000","last_fullsized":"2024-09-13T14:32:49.411568+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:00:16.248059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":1161,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.057082+0000","last_change":"2024-09-13T14:27:55.996934+0000","last_active":"2024-09-13T14:32:49.057082+0000","last_peered":"2024-09-13T14:32:49.057082+0000","last_clean":"2024-09-13T14:32:49.057082+0000","last_became_active":"2024-09-13T14:27:55.996422+0000","last_became_peered":"2024-09-13T14:27:55.996422+0000","last_unstale":"2024-09-13T14:32:49.057082+0000","last_undegraded":"2024-09-13T14:32:49.057082+0000","last_fullsized":"2024-09-13T14:32:49.057082+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:35.378769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"598'29","reported_seq":1145,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411775+0000","last_change":"2024-09-13T14:27:22.028245+0000","last_active":"2024-09-13T14:32:49.411775+0000","last_peered":"2024-09-13T14:32:49.411775+0000","last_clean":"2024-09-13T14:32:49.411775+0000","last_became_active":"2024-09-13T14:27:22.027881+0000","last_became_peered":"2024-09-13T14:27:22.027881+0000","last_unstale":"2024-09-13T14:32:49.411775+0000","last_undegraded":"2024-09-13T14:32:49.411775+0000","last_fullsized":"2024-09-13T14:32:49.411775+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":29,"log_dups_size":0,"ondisk_log_size":29,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:18.873740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5816,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":14,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":1083,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165110+0000","last_change":"2024-09-13T14:27:56.012367+0000","last_active":"2024-09-13T14:32:48.165110+0000","last_peered":"2024-09-13T14:32:48.165110+0000","last_clean":"2024-09-13T14:32:48.165110+0000","last_became_active":"2024-09-13T14:27:56.009400+0000","last_became_peered":"2024-09-13T14:27:56.009400+0000","last_unstale":"2024-09-13T14:32:48.165110+0000","last_undegraded":"2024-09-13T14:32:48.165110+0000","last_fullsized":"2024-09-13T14:32:48.165110+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:51.649547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":1294,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130035+0000","last_change":"2024-09-13T14:26:12.507005+0000","last_active":"2024-09-13T14:32:49.130035+0000","last_peered":"2024-09-13T14:32:49.130035+0000","last_clean":"2024-09-13T14:32:49.130035+0000","last_became_active":"2024-09-13T14:26:12.505837+0000","last_became_peered":"2024-09-13T14:26:12.505837+0000","last_unstale":"2024-09-13T14:32:49.130035+0000","last_undegraded":"2024-09-13T14:32:49.130035+0000","last_fullsized":"2024-09-13T14:32:49.130035+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:39.113757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1285,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.129916+0000","last_change":"2024-09-13T14:27:56.003276+0000","last_active":"2024-09-13T14:32:49.129916+0000","last_peered":"2024-09-13T14:32:49.129916+0000","last_clean":"2024-09-13T14:32:49.129916+0000","last_became_active":"2024-09-13T14:27:56.002973+0000","last_became_peered":"2024-09-13T14:27:56.002973+0000","last_unstale":"2024-09-13T14:32:49.129916+0000","last_undegraded":"2024-09-13T14:32:49.129916+0000","last_fullsized":"2024-09-13T14:32:49.129916+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:26:48.236105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":1153,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411767+0000","last_change":"2024-09-13T14:27:56.002343+0000","last_active":"2024-09-13T14:32:49.411767+0000","last_peered":"2024-09-13T14:32:49.411767+0000","last_clean":"2024-09-13T14:32:49.411767+0000","last_became_active":"2024-09-13T14:27:56.002021+0000","last_became_peered":"2024-09-13T14:27:56.002021+0000","last_unstale":"2024-09-13T14:32:49.411767+0000","last_undegraded":"2024-09-13T14:32:49.411767+0000","last_fullsized":"2024-09-13T14:32:49.411767+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:43.847203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":1083,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164615+0000","last_change":"2024-09-13T14:27:56.007587+0000","last_active":"2024-09-13T14:32:48.164615+0000","last_peered":"2024-09-13T14:32:48.164615+0000","last_clean":"2024-09-13T14:32:48.164615+0000","last_became_active":"2024-09-13T14:27:56.006995+0000","last_became_peered":"2024-09-13T14:27:56.006995+0000","last_unstale":"2024-09-13T14:32:48.164615+0000","last_undegraded":"2024-09-13T14:32:48.164615+0000","last_fullsized":"2024-09-13T14:32:48.164615+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:15.054725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":1118,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411188+0000","last_change":"2024-09-13T14:27:22.029973+0000","last_active":"2024-09-13T14:32:49.411188+0000","last_peered":"2024-09-13T14:32:49.411188+0000","last_clean":"2024-09-13T14:32:49.411188+0000","last_became_active":"2024-09-13T14:27:22.029606+0000","last_became_peered":"2024-09-13T14:27:22.029606+0000","last_unstale":"2024-09-13T14:32:49.411188+0000","last_undegraded":"2024-09-13T14:32:49.411188+0000","last_fullsized":"2024-09-13T14:32:49.411188+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:16:22.445530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":1107,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411181+0000","last_change":"2024-09-13T14:27:22.019943+0000","last_active":"2024-09-13T14:32:49.411181+0000","last_peered":"2024-09-13T14:32:49.411181+0000","last_clean":"2024-09-13T14:32:49.411181+0000","last_became_active":"2024-09-13T14:27:22.019133+0000","last_became_peered":"2024-09-13T14:27:22.019133+0000","last_unstale":"2024-09-13T14:32:49.411181+0000","last_undegraded":"2024-09-13T14:32:49.411181+0000","last_fullsized":"2024-09-13T14:32:49.411181+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:04:33.993105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":1262,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055277+0000","last_change":"2024-09-13T14:25:07.223925+0000","last_active":"2024-09-13T14:32:49.055277+0000","last_peered":"2024-09-13T14:32:49.055277+0000","last_clean":"2024-09-13T14:32:49.055277+0000","last_became_active":"2024-09-13T14:25:07.223576+0000","last_became_peered":"2024-09-13T14:25:07.223576+0000","last_unstale":"2024-09-13T14:32:49.055277+0000","last_undegraded":"2024-09-13T14:32:49.055277+0000","last_fullsized":"2024-09-13T14:32:49.055277+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:39.181191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1069,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164245+0000","last_change":"2024-09-13T14:27:56.001478+0000","last_active":"2024-09-13T14:32:48.164245+0000","last_peered":"2024-09-13T14:32:48.164245+0000","last_clean":"2024-09-13T14:32:48.164245+0000","last_became_active":"2024-09-13T14:27:56.001084+0000","last_became_peered":"2024-09-13T14:27:56.001084+0000","last_unstale":"2024-09-13T14:32:48.164245+0000","last_undegraded":"2024-09-13T14:32:48.164245+0000","last_fullsized":"2024-09-13T14:32:48.164245+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:04:34.800956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"844'790","reported_seq":4009,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055495+0000","last_change":"2024-09-13T14:32:48.161656+0000","last_active":"2024-09-13T14:32:49.055495+0000","last_peered":"2024-09-13T14:32:49.055495+0000","last_clean":"2024-09-13T14:32:49.055495+0000","last_became_active":"2024-09-13T14:25:07.224697+0000","last_became_peered":"2024-09-13T14:25:07.224697+0000","last_unstale":"2024-09-13T14:32:49.055495+0000","last_undegraded":"2024-09-13T14:32:49.055495+0000","last_fullsized":"2024-09-13T14:32:49.055495+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":790,"log_dups_size":0,"ondisk_log_size":790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:42.722340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001133614,"stat_sum":{"num_bytes":3063808,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":2,"num_read":152,"num_read_kb":253822,"num_write":283,"num_write_kb":81792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":46850048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.f","version":"57'1","reported_seq":1177,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.057244+0000","last_change":"2024-09-13T14:27:56.001364+0000","last_active":"2024-09-13T14:32:49.057244+0000","last_peered":"2024-09-13T14:32:49.057244+0000","last_clean":"2024-09-13T14:32:49.057244+0000","last_became_active":"2024-09-13T14:27:56.001031+0000","last_became_peered":"2024-09-13T14:27:56.001031+0000","last_unstale":"2024-09-13T14:32:49.057244+0000","last_undegraded":"2024-09-13T14:32:49.057244+0000","last_fullsized":"2024-09-13T14:32:49.057244+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:13.750977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":1083,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165437+0000","last_change":"2024-09-13T14:27:56.001682+0000","last_active":"2024-09-13T14:32:48.165437+0000","last_peered":"2024-09-13T14:32:48.165437+0000","last_clean":"2024-09-13T14:32:48.165437+0000","last_became_active":"2024-09-13T14:27:55.999947+0000","last_became_peered":"2024-09-13T14:27:55.999947+0000","last_unstale":"2024-09-13T14:32:48.165437+0000","last_undegraded":"2024-09-13T14:32:48.165437+0000","last_fullsized":"2024-09-13T14:32:48.165437+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:55:44.640159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"848'2189","reported_seq":5028,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.170782+0000","last_change":"2024-09-13T14:32:48.172803+0000","last_active":"2024-09-13T14:32:48.170782+0000","last_peered":"2024-09-13T14:32:48.170782+0000","last_clean":"2024-09-13T14:32:48.170782+0000","last_became_active":"2024-09-13T14:27:56.003264+0000","last_became_peered":"2024-09-13T14:27:56.003264+0000","last_unstale":"2024-09-13T14:32:48.170782+0000","last_undegraded":"2024-09-13T14:32:48.170782+0000","last_fullsized":"2024-09-13T14:32:48.170782+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":2189,"log_dups_size":0,"ondisk_log_size":2189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:12.354127+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0090583439999999994,"stat_sum":{"num_bytes":22192128,"num_objects":10,"num_object_clones":4,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":423,"num_read_kb":497462,"num_write":782,"num_write_kb":179352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":47251456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1d6","length":"2"},{"start":"1dc","length":"1"}]},{"pgid":"2.e","version":"57'2","reported_seq":1245,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055554+0000","last_change":"2024-09-13T14:27:56.006705+0000","last_active":"2024-09-13T14:32:49.055554+0000","last_peered":"2024-09-13T14:32:49.055554+0000","last_clean":"2024-09-13T14:32:49.055554+0000","last_became_active":"2024-09-13T14:27:56.006189+0000","last_became_peered":"2024-09-13T14:27:56.006189+0000","last_unstale":"2024-09-13T14:32:49.055554+0000","last_undegraded":"2024-09-13T14:32:49.055554+0000","last_fullsized":"2024-09-13T14:32:49.055554+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:03.571204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":1083,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.166007+0000","last_change":"2024-09-13T14:27:55.999850+0000","last_active":"2024-09-13T14:32:48.166007+0000","last_peered":"2024-09-13T14:32:48.166007+0000","last_clean":"2024-09-13T14:32:48.166007+0000","last_became_active":"2024-09-13T14:27:55.999529+0000","last_became_peered":"2024-09-13T14:27:55.999529+0000","last_unstale":"2024-09-13T14:32:48.166007+0000","last_undegraded":"2024-09-13T14:32:48.166007+0000","last_fullsized":"2024-09-13T14:32:48.166007+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:01.478935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"848'1489","reported_seq":4704,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055512+0000","last_change":"2024-09-13T14:32:48.169987+0000","last_active":"2024-09-13T14:32:49.055512+0000","last_peered":"2024-09-13T14:32:49.055512+0000","last_clean":"2024-09-13T14:32:49.055512+0000","last_became_active":"2024-09-13T14:26:12.500543+0000","last_became_peered":"2024-09-13T14:26:12.500543+0000","last_unstale":"2024-09-13T14:32:49.055512+0000","last_undegraded":"2024-09-13T14:32:49.055512+0000","last_fullsized":"2024-09-13T14:32:49.055512+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":1489,"log_dups_size":0,"ondisk_log_size":1489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:42.974642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0093319839999999998,"stat_sum":{"num_bytes":5586944,"num_objects":6,"num_object_clones":3,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":332,"num_read_kb":528500,"num_write":701,"num_write_kb":152352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":30883840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.d","version":"57'1","reported_seq":1270,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411637+0000","last_change":"2024-09-13T14:26:12.505112+0000","last_active":"2024-09-13T14:32:49.411637+0000","last_peered":"2024-09-13T14:32:49.411637+0000","last_clean":"2024-09-13T14:32:49.411637+0000","last_became_active":"2024-09-13T14:26:12.504742+0000","last_became_peered":"2024-09-13T14:26:12.504742+0000","last_unstale":"2024-09-13T14:32:49.411637+0000","last_undegraded":"2024-09-13T14:32:49.411637+0000","last_fullsized":"2024-09-13T14:32:49.411637+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:53.050492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1150,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411987+0000","last_change":"2024-09-13T14:27:22.021350+0000","last_active":"2024-09-13T14:32:49.411987+0000","last_peered":"2024-09-13T14:32:49.411987+0000","last_clean":"2024-09-13T14:32:49.411987+0000","last_became_active":"2024-09-13T14:27:22.021060+0000","last_became_peered":"2024-09-13T14:27:22.021060+0000","last_unstale":"2024-09-13T14:32:49.411987+0000","last_undegraded":"2024-09-13T14:32:49.411987+0000","last_fullsized":"2024-09-13T14:32:49.411987+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:13.587536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"848'2811","reported_seq":5892,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056816+0000","last_change":"2024-09-13T14:32:48.177011+0000","last_active":"2024-09-13T14:32:49.056816+0000","last_peered":"2024-09-13T14:32:49.056816+0000","last_clean":"2024-09-13T14:32:49.056816+0000","last_became_active":"2024-09-13T14:26:46.968287+0000","last_became_peered":"2024-09-13T14:26:46.968287+0000","last_unstale":"2024-09-13T14:32:49.056816+0000","last_undegraded":"2024-09-13T14:32:49.056816+0000","last_fullsized":"2024-09-13T14:32:49.056816+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":2811,"log_dups_size":0,"ondisk_log_size":2811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:19.676974+0000","scrub_duration":0,"objects_trimmed":2,"snaptrim_duration":0.016043734,"stat_sum":{"num_bytes":31514624,"num_objects":17,"num_object_clones":11,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":2,"num_read":527,"num_read_kb":660015,"num_write":1171,"num_write_kb":259352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":85540864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.c","version":"0'0","reported_seq":1250,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.057640+0000","last_change":"2024-09-13T14:26:46.968724+0000","last_active":"2024-09-13T14:32:49.057640+0000","last_peered":"2024-09-13T14:32:49.057640+0000","last_clean":"2024-09-13T14:32:49.057640+0000","last_became_active":"2024-09-13T14:26:46.968372+0000","last_became_peered":"2024-09-13T14:26:46.968372+0000","last_unstale":"2024-09-13T14:32:49.057640+0000","last_undegraded":"2024-09-13T14:32:49.057640+0000","last_fullsized":"2024-09-13T14:32:49.057640+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:49.406292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":1069,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.166024+0000","last_change":"2024-09-13T14:27:56.012980+0000","last_active":"2024-09-13T14:32:48.166024+0000","last_peered":"2024-09-13T14:32:48.166024+0000","last_clean":"2024-09-13T14:32:48.166024+0000","last_became_active":"2024-09-13T14:27:56.012004+0000","last_became_peered":"2024-09-13T14:27:56.012004+0000","last_unstale":"2024-09-13T14:32:48.166024+0000","last_undegraded":"2024-09-13T14:32:48.166024+0000","last_fullsized":"2024-09-13T14:32:48.166024+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:15:28.675184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":2889,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165623+0000","last_change":"2024-09-13T14:32:48.166002+0000","last_active":"2024-09-13T14:32:48.165623+0000","last_peered":"2024-09-13T14:32:48.165623+0000","last_clean":"2024-09-13T14:32:48.165623+0000","last_became_active":"2024-09-13T14:27:56.008133+0000","last_became_peered":"2024-09-13T14:27:56.008133+0000","last_unstale":"2024-09-13T14:32:48.165623+0000","last_undegraded":"2024-09-13T14:32:48.165623+0000","last_fullsized":"2024-09-13T14:32:48.165623+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:05.405172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061922799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1d6","length":"2"},{"start":"1dc","length":"1"}]},{"pgid":"4.5","version":"0'0","reported_seq":1107,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412585+0000","last_change":"2024-09-13T14:27:56.013831+0000","last_active":"2024-09-13T14:32:49.412585+0000","last_peered":"2024-09-13T14:32:49.412585+0000","last_clean":"2024-09-13T14:32:49.412585+0000","last_became_active":"2024-09-13T14:27:56.013317+0000","last_became_peered":"2024-09-13T14:27:56.013317+0000","last_unstale":"2024-09-13T14:32:49.412585+0000","last_undegraded":"2024-09-13T14:32:49.412585+0000","last_fullsized":"2024-09-13T14:32:49.412585+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:40:23.309796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055469799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"292'164","reported_seq":1413,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164264+0000","last_change":"2024-09-13T14:27:56.008532+0000","last_active":"2024-09-13T14:32:48.164264+0000","last_peered":"2024-09-13T14:32:48.164264+0000","last_clean":"2024-09-13T14:32:48.164264+0000","last_became_active":"2024-09-13T14:27:56.006781+0000","last_became_peered":"2024-09-13T14:27:56.006781+0000","last_unstale":"2024-09-13T14:32:48.164264+0000","last_undegraded":"2024-09-13T14:32:48.164264+0000","last_fullsized":"2024-09-13T14:32:48.164264+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:56.059430+0000","last_clean_scrub_stamp":"2024-09-13T14:12:56.059430+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:52.885528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":876,"num_read_kb":1739,"num_write":277,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1162,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412087+0000","last_change":"2024-09-13T14:27:56.005727+0000","last_active":"2024-09-13T14:32:49.412087+0000","last_peered":"2024-09-13T14:32:49.412087+0000","last_clean":"2024-09-13T14:32:49.412087+0000","last_became_active":"2024-09-13T14:27:56.005221+0000","last_became_peered":"2024-09-13T14:27:56.005221+0000","last_unstale":"2024-09-13T14:32:49.412087+0000","last_undegraded":"2024-09-13T14:32:49.412087+0000","last_fullsized":"2024-09-13T14:32:49.412087+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:54.610351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1207,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143537+0000","last_change":"2024-09-13T14:27:22.034808+0000","last_active":"2024-09-13T14:32:46.143537+0000","last_peered":"2024-09-13T14:32:46.143537+0000","last_clean":"2024-09-13T14:32:46.143537+0000","last_became_active":"2024-09-13T14:27:22.034571+0000","last_became_peered":"2024-09-13T14:27:22.034571+0000","last_unstale":"2024-09-13T14:32:46.143537+0000","last_undegraded":"2024-09-13T14:32:46.143537+0000","last_fullsized":"2024-09-13T14:32:46.143537+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:44.082041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"848'789","reported_seq":3837,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.413231+0000","last_change":"2024-09-13T14:32:48.177619+0000","last_active":"2024-09-13T14:32:49.413231+0000","last_peered":"2024-09-13T14:32:49.413231+0000","last_clean":"2024-09-13T14:32:49.413231+0000","last_became_active":"2024-09-13T14:26:46.972825+0000","last_became_peered":"2024-09-13T14:26:46.972825+0000","last_unstale":"2024-09-13T14:32:49.413231+0000","last_undegraded":"2024-09-13T14:32:49.413231+0000","last_fullsized":"2024-09-13T14:32:49.413231+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":789,"log_dups_size":0,"ondisk_log_size":789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:12.997504+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.010086807,"stat_sum":{"num_bytes":19693568,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":144,"num_read_kb":315531,"num_write":329,"num_write_kb":89440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":9134080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.6","version":"0'0","reported_seq":1296,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130220+0000","last_change":"2024-09-13T14:25:07.217774+0000","last_active":"2024-09-13T14:32:49.130220+0000","last_peered":"2024-09-13T14:32:49.130220+0000","last_clean":"2024-09-13T14:32:49.130220+0000","last_became_active":"2024-09-13T14:25:07.217463+0000","last_became_peered":"2024-09-13T14:25:07.217463+0000","last_unstale":"2024-09-13T14:32:49.130220+0000","last_undegraded":"2024-09-13T14:32:49.130220+0000","last_fullsized":"2024-09-13T14:32:49.130220+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:58.402004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1072,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164615+0000","last_change":"2024-09-13T14:27:55.999210+0000","last_active":"2024-09-13T14:32:48.164615+0000","last_peered":"2024-09-13T14:32:48.164615+0000","last_clean":"2024-09-13T14:32:48.164615+0000","last_became_active":"2024-09-13T14:27:55.998868+0000","last_became_peered":"2024-09-13T14:27:55.998868+0000","last_unstale":"2024-09-13T14:32:48.164615+0000","last_undegraded":"2024-09-13T14:32:48.164615+0000","last_fullsized":"2024-09-13T14:32:48.164615+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:55.073359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1300,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.129726+0000","last_change":"2024-09-13T14:25:39.980920+0000","last_active":"2024-09-13T14:32:49.129726+0000","last_peered":"2024-09-13T14:32:49.129726+0000","last_clean":"2024-09-13T14:32:49.129726+0000","last_became_active":"2024-09-13T14:25:39.979925+0000","last_became_peered":"2024-09-13T14:25:39.979925+0000","last_unstale":"2024-09-13T14:32:49.129726+0000","last_undegraded":"2024-09-13T14:32:49.129726+0000","last_fullsized":"2024-09-13T14:32:49.129726+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:04.637933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"848'2139","reported_seq":5285,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055914+0000","last_change":"2024-09-13T14:32:48.187146+0000","last_active":"2024-09-13T14:32:49.055914+0000","last_peered":"2024-09-13T14:32:49.055914+0000","last_clean":"2024-09-13T14:32:49.055914+0000","last_became_active":"2024-09-13T14:26:46.968210+0000","last_became_peered":"2024-09-13T14:26:46.968210+0000","last_unstale":"2024-09-13T14:32:49.055914+0000","last_undegraded":"2024-09-13T14:32:49.055914+0000","last_fullsized":"2024-09-13T14:32:49.055914+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":2139,"log_dups_size":0,"ondisk_log_size":2139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:22:56.997559+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.025303048000000002,"stat_sum":{"num_bytes":27590656,"num_objects":13,"num_object_clones":8,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":1,"num_read":402,"num_read_kb":850128,"num_write":873,"num_write_kb":226408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":50364416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.7","version":"0'0","reported_seq":1271,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411792+0000","last_change":"2024-09-13T14:26:46.975632+0000","last_active":"2024-09-13T14:32:49.411792+0000","last_peered":"2024-09-13T14:32:49.411792+0000","last_clean":"2024-09-13T14:32:49.411792+0000","last_became_active":"2024-09-13T14:26:46.974820+0000","last_became_peered":"2024-09-13T14:26:46.974820+0000","last_unstale":"2024-09-13T14:32:49.411792+0000","last_undegraded":"2024-09-13T14:32:49.411792+0000","last_fullsized":"2024-09-13T14:32:49.411792+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:52:19.045917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034207600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1250,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.055440+0000","last_change":"2024-09-13T14:25:39.978169+0000","last_active":"2024-09-13T14:32:49.055440+0000","last_peered":"2024-09-13T14:32:49.055440+0000","last_clean":"2024-09-13T14:32:49.055440+0000","last_became_active":"2024-09-13T14:25:39.977782+0000","last_became_peered":"2024-09-13T14:25:39.977782+0000","last_unstale":"2024-09-13T14:32:49.055440+0000","last_undegraded":"2024-09-13T14:32:49.055440+0000","last_fullsized":"2024-09-13T14:32:49.055440+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:45.707488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1265,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411200+0000","last_change":"2024-09-13T14:27:22.028307+0000","last_active":"2024-09-13T14:32:49.411200+0000","last_peered":"2024-09-13T14:32:49.411200+0000","last_clean":"2024-09-13T14:32:49.411200+0000","last_became_active":"2024-09-13T14:27:22.027926+0000","last_became_peered":"2024-09-13T14:27:22.027926+0000","last_unstale":"2024-09-13T14:32:49.411200+0000","last_undegraded":"2024-09-13T14:32:49.411200+0000","last_fullsized":"2024-09-13T14:32:49.411200+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:17.066293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"848'1560","reported_seq":4860,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130477+0000","last_change":"2024-09-13T14:32:48.180454+0000","last_active":"2024-09-13T14:32:49.130477+0000","last_peered":"2024-09-13T14:32:49.130477+0000","last_clean":"2024-09-13T14:32:49.130477+0000","last_became_active":"2024-09-13T14:26:12.510174+0000","last_became_peered":"2024-09-13T14:26:12.510174+0000","last_unstale":"2024-09-13T14:32:49.130477+0000","last_undegraded":"2024-09-13T14:32:49.130477+0000","last_fullsized":"2024-09-13T14:32:49.130477+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":1560,"log_dups_size":0,"ondisk_log_size":1560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:16.537991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.012212557000000001,"stat_sum":{"num_bytes":31195136,"num_objects":7,"num_object_clones":3,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":304,"num_read_kb":423097,"num_write":649,"num_write_kb":167536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":46489600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.4","version":"0'0","reported_seq":1274,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411763+0000","last_change":"2024-09-13T14:25:39.985574+0000","last_active":"2024-09-13T14:32:49.411763+0000","last_peered":"2024-09-13T14:32:49.411763+0000","last_clean":"2024-09-13T14:32:49.411763+0000","last_became_active":"2024-09-13T14:25:39.985252+0000","last_became_peered":"2024-09-13T14:25:39.985252+0000","last_unstale":"2024-09-13T14:32:49.411763+0000","last_undegraded":"2024-09-13T14:32:49.411763+0000","last_fullsized":"2024-09-13T14:32:49.411763+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:52.002109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1129,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411559+0000","last_change":"2024-09-13T14:27:22.023190+0000","last_active":"2024-09-13T14:32:49.411559+0000","last_peered":"2024-09-13T14:32:49.411559+0000","last_clean":"2024-09-13T14:32:49.411559+0000","last_became_active":"2024-09-13T14:27:22.022900+0000","last_became_peered":"2024-09-13T14:27:22.022900+0000","last_unstale":"2024-09-13T14:32:49.411559+0000","last_undegraded":"2024-09-13T14:32:49.411559+0000","last_fullsized":"2024-09-13T14:32:49.411559+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:45.476429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1173,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056556+0000","last_change":"2024-09-13T14:27:22.021495+0000","last_active":"2024-09-13T14:32:49.056556+0000","last_peered":"2024-09-13T14:32:49.056556+0000","last_clean":"2024-09-13T14:32:49.056556+0000","last_became_active":"2024-09-13T14:27:22.020979+0000","last_became_peered":"2024-09-13T14:27:22.020979+0000","last_unstale":"2024-09-13T14:32:49.056556+0000","last_undegraded":"2024-09-13T14:32:49.056556+0000","last_fullsized":"2024-09-13T14:32:49.056556+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:15.142574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"837'422","reported_seq":3780,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130379+0000","last_change":"2024-09-13T14:32:48.173505+0000","last_active":"2024-09-13T14:32:49.130379+0000","last_peered":"2024-09-13T14:32:49.130379+0000","last_clean":"2024-09-13T14:32:49.130379+0000","last_became_active":"2024-09-13T14:27:22.020183+0000","last_became_peered":"2024-09-13T14:27:22.020183+0000","last_unstale":"2024-09-13T14:32:49.130379+0000","last_undegraded":"2024-09-13T14:32:49.130379+0000","last_fullsized":"2024-09-13T14:32:49.130379+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":422,"log_dups_size":0,"ondisk_log_size":422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:43.292231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014304789999999999,"stat_sum":{"num_bytes":4833280,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":1,"num_read":86,"num_read_kb":150541,"num_write":119,"num_write_kb":37144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.2","version":"66'2","reported_seq":1276,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411464+0000","last_change":"2024-09-13T14:26:46.967216+0000","last_active":"2024-09-13T14:32:49.411464+0000","last_peered":"2024-09-13T14:32:49.411464+0000","last_clean":"2024-09-13T14:32:49.411464+0000","last_became_active":"2024-09-13T14:26:46.966846+0000","last_became_peered":"2024-09-13T14:26:46.966846+0000","last_unstale":"2024-09-13T14:32:49.411464+0000","last_undegraded":"2024-09-13T14:32:49.411464+0000","last_fullsized":"2024-09-13T14:32:49.411464+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:30.368427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1277,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411492+0000","last_change":"2024-09-13T14:27:55.999917+0000","last_active":"2024-09-13T14:32:49.411492+0000","last_peered":"2024-09-13T14:32:49.411492+0000","last_clean":"2024-09-13T14:32:49.411492+0000","last_became_active":"2024-09-13T14:27:55.999644+0000","last_became_peered":"2024-09-13T14:27:55.999644+0000","last_unstale":"2024-09-13T14:32:49.411492+0000","last_undegraded":"2024-09-13T14:32:49.411492+0000","last_fullsized":"2024-09-13T14:32:49.411492+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:25.704623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1156,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412705+0000","last_change":"2024-09-13T14:26:46.968853+0000","last_active":"2024-09-13T14:32:49.412705+0000","last_peered":"2024-09-13T14:32:49.412705+0000","last_clean":"2024-09-13T14:32:49.412705+0000","last_became_active":"2024-09-13T14:26:46.968495+0000","last_became_peered":"2024-09-13T14:26:46.968495+0000","last_unstale":"2024-09-13T14:32:49.412705+0000","last_undegraded":"2024-09-13T14:32:49.412705+0000","last_fullsized":"2024-09-13T14:32:49.412705+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:49:12.181114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"841'1086","reported_seq":4015,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411854+0000","last_change":"2024-09-13T14:32:48.170907+0000","last_active":"2024-09-13T14:32:49.411854+0000","last_peered":"2024-09-13T14:32:49.411854+0000","last_clean":"2024-09-13T14:32:49.411854+0000","last_became_active":"2024-09-13T14:27:22.026041+0000","last_became_peered":"2024-09-13T14:27:22.026041+0000","last_unstale":"2024-09-13T14:32:49.411854+0000","last_undegraded":"2024-09-13T14:32:49.411854+0000","last_fullsized":"2024-09-13T14:32:49.411854+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":1086,"log_dups_size":0,"ondisk_log_size":1086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:37.640456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011973559999999999,"stat_sum":{"num_bytes":7692288,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":193,"num_read_kb":223087,"num_write":425,"num_write_kb":101672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":3850240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.3","version":"64'1","reported_seq":1297,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130148+0000","last_change":"2024-09-13T14:27:56.003573+0000","last_active":"2024-09-13T14:32:49.130148+0000","last_peered":"2024-09-13T14:32:49.130148+0000","last_clean":"2024-09-13T14:32:49.130148+0000","last_became_active":"2024-09-13T14:27:56.003234+0000","last_became_peered":"2024-09-13T14:27:56.003234+0000","last_unstale":"2024-09-13T14:32:49.130148+0000","last_undegraded":"2024-09-13T14:32:49.130148+0000","last_fullsized":"2024-09-13T14:32:49.130148+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:11:46.420437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1086,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165518+0000","last_change":"2024-09-13T14:27:56.008417+0000","last_active":"2024-09-13T14:32:48.165518+0000","last_peered":"2024-09-13T14:32:48.165518+0000","last_clean":"2024-09-13T14:32:48.165518+0000","last_became_active":"2024-09-13T14:27:56.001369+0000","last_became_peered":"2024-09-13T14:27:56.001369+0000","last_unstale":"2024-09-13T14:32:48.165518+0000","last_undegraded":"2024-09-13T14:32:48.165518+0000","last_fullsized":"2024-09-13T14:32:48.165518+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:32:40.020109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1265,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411809+0000","last_change":"2024-09-13T14:26:46.969335+0000","last_active":"2024-09-13T14:32:49.411809+0000","last_peered":"2024-09-13T14:32:49.411809+0000","last_clean":"2024-09-13T14:32:49.411809+0000","last_became_active":"2024-09-13T14:26:46.968921+0000","last_became_peered":"2024-09-13T14:26:46.968921+0000","last_unstale":"2024-09-13T14:32:49.411809+0000","last_undegraded":"2024-09-13T14:32:49.411809+0000","last_fullsized":"2024-09-13T14:32:49.411809+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:14.800780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"848'1202","reported_seq":4311,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056556+0000","last_change":"2024-09-13T14:32:48.173592+0000","last_active":"2024-09-13T14:32:49.056556+0000","last_peered":"2024-09-13T14:32:49.056556+0000","last_clean":"2024-09-13T14:32:49.056556+0000","last_became_active":"2024-09-13T14:27:55.994277+0000","last_became_peered":"2024-09-13T14:27:55.994277+0000","last_unstale":"2024-09-13T14:32:49.056556+0000","last_undegraded":"2024-09-13T14:32:49.056556+0000","last_fullsized":"2024-09-13T14:32:49.056556+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":1202,"log_dups_size":0,"ondisk_log_size":1202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:44.041855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.013015861,"stat_sum":{"num_bytes":24510464,"num_objects":6,"num_object_clones":3,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":1,"num_read":240,"num_read_kb":773477,"num_write":416,"num_write_kb":144776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":41025536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.0","version":"0'0","reported_seq":1216,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.142813+0000","last_change":"2024-09-13T14:27:55.999869+0000","last_active":"2024-09-13T14:32:46.142813+0000","last_peered":"2024-09-13T14:32:46.142813+0000","last_clean":"2024-09-13T14:32:46.142813+0000","last_became_active":"2024-09-13T14:27:55.999522+0000","last_became_peered":"2024-09-13T14:27:55.999522+0000","last_unstale":"2024-09-13T14:32:46.142813+0000","last_undegraded":"2024-09-13T14:32:46.142813+0000","last_fullsized":"2024-09-13T14:32:46.142813+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:21:08.196743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049448099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"600'2","reported_seq":1271,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411757+0000","last_change":"2024-09-13T14:27:22.030267+0000","last_active":"2024-09-13T14:32:49.411757+0000","last_peered":"2024-09-13T14:32:49.411757+0000","last_clean":"2024-09-13T14:32:49.411757+0000","last_became_active":"2024-09-13T14:27:22.029308+0000","last_became_peered":"2024-09-13T14:27:22.029308+0000","last_unstale":"2024-09-13T14:32:49.411757+0000","last_undegraded":"2024-09-13T14:32:49.411757+0000","last_fullsized":"2024-09-13T14:32:49.411757+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:17.241289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":1218,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143368+0000","last_change":"2024-09-13T14:27:55.999515+0000","last_active":"2024-09-13T14:32:46.143368+0000","last_peered":"2024-09-13T14:32:46.143368+0000","last_clean":"2024-09-13T14:32:46.143368+0000","last_became_active":"2024-09-13T14:27:55.999194+0000","last_became_peered":"2024-09-13T14:27:55.999194+0000","last_unstale":"2024-09-13T14:32:46.143368+0000","last_undegraded":"2024-09-13T14:32:46.143368+0000","last_fullsized":"2024-09-13T14:32:46.143368+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:54:44.016975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"841'396","reported_seq":3576,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143279+0000","last_change":"2024-09-13T14:32:44.062448+0000","last_active":"2024-09-13T14:32:46.143279+0000","last_peered":"2024-09-13T14:32:46.143279+0000","last_clean":"2024-09-13T14:32:46.143279+0000","last_became_active":"2024-09-13T14:26:12.502067+0000","last_became_peered":"2024-09-13T14:26:12.502067+0000","last_unstale":"2024-09-13T14:32:46.143279+0000","last_undegraded":"2024-09-13T14:32:46.143279+0000","last_fullsized":"2024-09-13T14:32:46.143279+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":396,"log_dups_size":0,"ondisk_log_size":396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:03.258272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028166279999999998,"stat_sum":{"num_bytes":4341760,"num_objects":4,"num_object_clones":3,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":87,"num_read_kb":143543,"num_write":207,"num_write_kb":48040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"1d6","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"4.1","version":"0'0","reported_seq":1172,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056628+0000","last_change":"2024-09-13T14:27:22.021673+0000","last_active":"2024-09-13T14:32:49.056628+0000","last_peered":"2024-09-13T14:32:49.056628+0000","last_clean":"2024-09-13T14:32:49.056628+0000","last_became_active":"2024-09-13T14:27:22.021351+0000","last_became_peered":"2024-09-13T14:27:22.021351+0000","last_unstale":"2024-09-13T14:32:49.056628+0000","last_undegraded":"2024-09-13T14:32:49.056628+0000","last_fullsized":"2024-09-13T14:32:49.056628+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:16:20.359872+0000","last_clean_scrub_stamp":"2024-09-13T14:16:20.359872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:35.114375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":1095,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411761+0000","last_change":"2024-09-13T14:27:56.010328+0000","last_active":"2024-09-13T14:32:49.411761+0000","last_peered":"2024-09-13T14:32:49.411761+0000","last_clean":"2024-09-13T14:32:49.411761+0000","last_became_active":"2024-09-13T14:27:56.009853+0000","last_became_peered":"2024-09-13T14:27:56.009853+0000","last_unstale":"2024-09-13T14:32:49.411761+0000","last_undegraded":"2024-09-13T14:32:49.411761+0000","last_fullsized":"2024-09-13T14:32:49.411761+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:32.300853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1234,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056069+0000","last_change":"2024-09-13T14:26:12.507203+0000","last_active":"2024-09-13T14:32:49.056069+0000","last_peered":"2024-09-13T14:32:49.056069+0000","last_clean":"2024-09-13T14:32:49.056069+0000","last_became_active":"2024-09-13T14:26:12.506864+0000","last_became_peered":"2024-09-13T14:26:12.506864+0000","last_unstale":"2024-09-13T14:32:49.056069+0000","last_undegraded":"2024-09-13T14:32:49.056069+0000","last_fullsized":"2024-09-13T14:32:49.056069+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:06.988134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.f","version":"848'1218","reported_seq":4220,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412877+0000","last_change":"2024-09-13T14:32:48.177365+0000","last_active":"2024-09-13T14:32:49.412877+0000","last_peered":"2024-09-13T14:32:49.412877+0000","last_clean":"2024-09-13T14:32:49.412877+0000","last_became_active":"2024-09-13T14:27:22.024577+0000","last_became_peered":"2024-09-13T14:27:22.024577+0000","last_unstale":"2024-09-13T14:32:49.412877+0000","last_undegraded":"2024-09-13T14:32:49.412877+0000","last_fullsized":"2024-09-13T14:32:49.412877+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":1218,"log_dups_size":0,"ondisk_log_size":1218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:55.191509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0088549690000000007,"stat_sum":{"num_bytes":15048704,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":1,"num_read":210,"num_read_kb":498841,"num_write":645,"num_write_kb":137288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":25206784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.8","version":"0'0","reported_seq":1072,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164439+0000","last_change":"2024-09-13T14:27:55.998702+0000","last_active":"2024-09-13T14:32:48.164439+0000","last_peered":"2024-09-13T14:32:48.164439+0000","last_clean":"2024-09-13T14:32:48.164439+0000","last_became_active":"2024-09-13T14:27:55.998439+0000","last_became_peered":"2024-09-13T14:27:55.998439+0000","last_unstale":"2024-09-13T14:32:48.164439+0000","last_undegraded":"2024-09-13T14:32:48.164439+0000","last_fullsized":"2024-09-13T14:32:48.164439+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:24.747143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1198,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056850+0000","last_change":"2024-09-13T14:27:55.994476+0000","last_active":"2024-09-13T14:32:49.056850+0000","last_peered":"2024-09-13T14:32:49.056850+0000","last_clean":"2024-09-13T14:32:49.056850+0000","last_became_active":"2024-09-13T14:27:55.994188+0000","last_became_peered":"2024-09-13T14:27:55.994188+0000","last_unstale":"2024-09-13T14:32:49.056850+0000","last_undegraded":"2024-09-13T14:32:49.056850+0000","last_fullsized":"2024-09-13T14:32:49.056850+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:01.839895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"837'334","reported_seq":3478,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.057228+0000","last_change":"2024-09-13T14:32:48.161718+0000","last_active":"2024-09-13T14:32:49.057228+0000","last_peered":"2024-09-13T14:32:49.057228+0000","last_clean":"2024-09-13T14:32:49.057228+0000","last_became_active":"2024-09-13T14:26:46.965602+0000","last_became_peered":"2024-09-13T14:26:46.965602+0000","last_unstale":"2024-09-13T14:32:49.057228+0000","last_undegraded":"2024-09-13T14:32:49.057228+0000","last_fullsized":"2024-09-13T14:32:49.057228+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":334,"log_dups_size":0,"ondisk_log_size":334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:21.044090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039918700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":72,"num_read_kb":113997,"num_write":174,"num_write_kb":55040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":3031040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.9","version":"0'0","reported_seq":1263,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411158+0000","last_change":"2024-09-13T14:27:55.999203+0000","last_active":"2024-09-13T14:32:49.411158+0000","last_peered":"2024-09-13T14:32:49.411158+0000","last_clean":"2024-09-13T14:32:49.411158+0000","last_became_active":"2024-09-13T14:27:55.998906+0000","last_became_peered":"2024-09-13T14:27:55.998906+0000","last_unstale":"2024-09-13T14:32:49.411158+0000","last_undegraded":"2024-09-13T14:32:49.411158+0000","last_fullsized":"2024-09-13T14:32:49.411158+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:58.327173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1209,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143057+0000","last_change":"2024-09-13T14:27:56.000604+0000","last_active":"2024-09-13T14:32:46.143057+0000","last_peered":"2024-09-13T14:32:46.143057+0000","last_clean":"2024-09-13T14:32:46.143057+0000","last_became_active":"2024-09-13T14:27:56.000273+0000","last_became_peered":"2024-09-13T14:27:56.000273+0000","last_unstale":"2024-09-13T14:32:46.143057+0000","last_undegraded":"2024-09-13T14:32:46.143057+0000","last_fullsized":"2024-09-13T14:32:46.143057+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:08.897484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"848'903","reported_seq":4091,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.057116+0000","last_change":"2024-09-13T14:32:48.178671+0000","last_active":"2024-09-13T14:32:49.057116+0000","last_peered":"2024-09-13T14:32:49.057116+0000","last_clean":"2024-09-13T14:32:49.057116+0000","last_became_active":"2024-09-13T14:27:56.013346+0000","last_became_peered":"2024-09-13T14:27:56.013346+0000","last_unstale":"2024-09-13T14:32:49.057116+0000","last_undegraded":"2024-09-13T14:32:49.057116+0000","last_fullsized":"2024-09-13T14:32:49.057116+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":903,"log_dups_size":0,"ondisk_log_size":903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:52.909318+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.017467149000000001,"stat_sum":{"num_bytes":5832704,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":158,"num_read_kb":180304,"num_write":440,"num_write_kb":66336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":15679488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.a","version":"0'0","reported_seq":1263,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411236+0000","last_change":"2024-09-13T14:27:55.993390+0000","last_active":"2024-09-13T14:32:49.411236+0000","last_peered":"2024-09-13T14:32:49.411236+0000","last_clean":"2024-09-13T14:32:49.411236+0000","last_became_active":"2024-09-13T14:27:55.993070+0000","last_became_peered":"2024-09-13T14:27:55.993070+0000","last_unstale":"2024-09-13T14:32:49.411236+0000","last_undegraded":"2024-09-13T14:32:49.411236+0000","last_fullsized":"2024-09-13T14:32:49.411236+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:30:12.285192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":1218,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.144618+0000","last_change":"2024-09-13T14:26:12.508842+0000","last_active":"2024-09-13T14:32:46.144618+0000","last_peered":"2024-09-13T14:32:46.144618+0000","last_clean":"2024-09-13T14:32:46.144618+0000","last_became_active":"2024-09-13T14:26:12.508334+0000","last_became_peered":"2024-09-13T14:26:12.508334+0000","last_unstale":"2024-09-13T14:32:46.144618+0000","last_undegraded":"2024-09-13T14:32:46.144618+0000","last_fullsized":"2024-09-13T14:32:46.144618+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:12.713957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"848'2227","reported_seq":5406,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.411458+0000","last_change":"2024-09-13T14:32:48.185632+0000","last_active":"2024-09-13T14:32:49.411458+0000","last_peered":"2024-09-13T14:32:49.411458+0000","last_clean":"2024-09-13T14:32:49.411458+0000","last_became_active":"2024-09-13T14:26:12.514668+0000","last_became_peered":"2024-09-13T14:26:12.514668+0000","last_unstale":"2024-09-13T14:32:49.411458+0000","last_undegraded":"2024-09-13T14:32:49.411458+0000","last_fullsized":"2024-09-13T14:32:49.411458+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:18.461731+0000","last_clean_scrub_stamp":"2024-09-13T14:19:18.461731+0000","objects_scrubbed":0,"log_size":2227,"log_dups_size":0,"ondisk_log_size":2227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:05.895177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.010598773000000001,"stat_sum":{"num_bytes":29573120,"num_objects":14,"num_object_clones":9,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":1,"num_read":417,"num_read_kb":583000,"num_write":1145,"num_write_kb":209016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":18145280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"1d7","length":"1"},{"start":"1dc","length":"1"}]},{"pgid":"2.b","version":"598'3","reported_seq":1083,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164458+0000","last_change":"2024-09-13T14:27:56.008630+0000","last_active":"2024-09-13T14:32:48.164458+0000","last_peered":"2024-09-13T14:32:48.164458+0000","last_clean":"2024-09-13T14:32:48.164458+0000","last_became_active":"2024-09-13T14:27:56.008036+0000","last_became_peered":"2024-09-13T14:27:56.008036+0000","last_unstale":"2024-09-13T14:32:48.164458+0000","last_undegraded":"2024-09-13T14:32:48.164458+0000","last_fullsized":"2024-09-13T14:32:48.164458+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:07:19.209022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1115,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412029+0000","last_change":"2024-09-13T14:27:22.032896+0000","last_active":"2024-09-13T14:32:49.412029+0000","last_peered":"2024-09-13T14:32:49.412029+0000","last_clean":"2024-09-13T14:32:49.412029+0000","last_became_active":"2024-09-13T14:27:22.032311+0000","last_became_peered":"2024-09-13T14:27:22.032311+0000","last_unstale":"2024-09-13T14:32:49.412029+0000","last_undegraded":"2024-09-13T14:32:49.412029+0000","last_fullsized":"2024-09-13T14:32:49.412029+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:39.711022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":1139,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.412458+0000","last_change":"2024-09-13T14:27:22.023552+0000","last_active":"2024-09-13T14:32:49.412458+0000","last_peered":"2024-09-13T14:32:49.412458+0000","last_clean":"2024-09-13T14:32:49.412458+0000","last_became_active":"2024-09-13T14:27:22.023266+0000","last_became_peered":"2024-09-13T14:27:22.023266+0000","last_unstale":"2024-09-13T14:32:49.412458+0000","last_undegraded":"2024-09-13T14:32:49.412458+0000","last_fullsized":"2024-09-13T14:32:49.412458+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:49.165085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":1082,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.164932+0000","last_change":"2024-09-13T14:27:56.008429+0000","last_active":"2024-09-13T14:32:48.164932+0000","last_peered":"2024-09-13T14:32:48.164932+0000","last_clean":"2024-09-13T14:32:48.164932+0000","last_became_active":"2024-09-13T14:27:56.007959+0000","last_became_peered":"2024-09-13T14:27:56.007959+0000","last_unstale":"2024-09-13T14:32:48.164932+0000","last_undegraded":"2024-09-13T14:32:48.164932+0000","last_fullsized":"2024-09-13T14:32:48.164932+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:12.814645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1234,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.056844+0000","last_change":"2024-09-13T14:26:12.508208+0000","last_active":"2024-09-13T14:32:49.056844+0000","last_peered":"2024-09-13T14:32:49.056844+0000","last_clean":"2024-09-13T14:32:49.056844+0000","last_became_active":"2024-09-13T14:26:12.507862+0000","last_became_peered":"2024-09-13T14:26:12.507862+0000","last_unstale":"2024-09-13T14:32:49.056844+0000","last_undegraded":"2024-09-13T14:32:49.056844+0000","last_fullsized":"2024-09-13T14:32:49.056844+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:41.534891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":1075,"reported_epoch":848,"state":"active+clean","last_fresh":"2024-09-13T14:32:48.165685+0000","last_change":"2024-09-13T14:27:56.012050+0000","last_active":"2024-09-13T14:32:48.165685+0000","last_peered":"2024-09-13T14:32:48.165685+0000","last_clean":"2024-09-13T14:32:48.165685+0000","last_became_active":"2024-09-13T14:27:56.002066+0000","last_became_peered":"2024-09-13T14:27:56.002066+0000","last_unstale":"2024-09-13T14:32:48.165685+0000","last_undegraded":"2024-09-13T14:32:48.165685+0000","last_fullsized":"2024-09-13T14:32:48.165685+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:14.905434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":1286,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130292+0000","last_change":"2024-09-13T14:26:46.971517+0000","last_active":"2024-09-13T14:32:49.130292+0000","last_peered":"2024-09-13T14:32:49.130292+0000","last_clean":"2024-09-13T14:32:49.130292+0000","last_became_active":"2024-09-13T14:26:46.971178+0000","last_became_peered":"2024-09-13T14:26:46.971178+0000","last_unstale":"2024-09-13T14:32:49.130292+0000","last_undegraded":"2024-09-13T14:32:49.130292+0000","last_fullsized":"2024-09-13T14:32:49.130292+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:25.282554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"598'2","reported_seq":1223,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143728+0000","last_change":"2024-09-13T14:26:46.970827+0000","last_active":"2024-09-13T14:32:46.143728+0000","last_peered":"2024-09-13T14:32:46.143728+0000","last_clean":"2024-09-13T14:32:46.143728+0000","last_became_active":"2024-09-13T14:26:46.970201+0000","last_became_peered":"2024-09-13T14:26:46.970201+0000","last_unstale":"2024-09-13T14:32:46.143728+0000","last_undegraded":"2024-09-13T14:32:46.143728+0000","last_fullsized":"2024-09-13T14:32:46.143728+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:34.950890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":1201,"reported_epoch":846,"state":"active+clean","last_fresh":"2024-09-13T14:32:46.143626+0000","last_change":"2024-09-13T14:27:22.029494+0000","last_active":"2024-09-13T14:32:46.143626+0000","last_peered":"2024-09-13T14:32:46.143626+0000","last_clean":"2024-09-13T14:32:46.143626+0000","last_became_active":"2024-09-13T14:27:22.029156+0000","last_became_peered":"2024-09-13T14:27:22.029156+0000","last_unstale":"2024-09-13T14:32:46.143626+0000","last_undegraded":"2024-09-13T14:32:46.143626+0000","last_fullsized":"2024-09-13T14:32:46.143626+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:25.534586+0000","last_clean_scrub_stamp":"2024-09-13T14:15:25.534586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:21:54.034981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"598'5","reported_seq":1316,"reported_epoch":849,"state":"active+clean","last_fresh":"2024-09-13T14:32:49.130543+0000","last_change":"2024-09-13T14:26:12.506171+0000","last_active":"2024-09-13T14:32:49.130543+0000","last_peered":"2024-09-13T14:32:49.130543+0000","last_clean":"2024-09-13T14:32:49.130543+0000","last_became_active":"2024-09-13T14:26:12.505742+0000","last_became_peered":"2024-09-13T14:26:12.505742+0000","last_unstale":"2024-09-13T14:32:49.130543+0000","last_undegraded":"2024-09-13T14:32:49.130543+0000","last_fullsized":"2024-09-13T14:32:49.130543+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:21.528831+0000","last_clean_scrub_stamp":"2024-09-13T14:15:21.528831+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:43:50.582400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":232669184,"num_objects":102,"num_object_clones":57,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":9,"num_read":3747,"num_read_kb":6195345,"num_write":8359,"num_write_kb":1955544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":437149696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":338694144,"data_stored":576143360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19555,"ondisk_log_size":19555,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6602,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":79,"num_read_kb":86,"num_write":68,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47476,"internal_metadata":0},"log_size":68,"ondisk_log_size":68,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":876,"num_read_kb":1739,"num_write":277,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164,"ondisk_log_size":164,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":3,"up_from":443,"seq":1902670512215,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":144592,"kb_used_data":31680,"kb_used_omap":18,"kb_used_meta":112877,"kb_avail":93600560,"statfs":{"total":95995035648,"available":95846973440,"internally_reserved":0,"allocated":32440320,"data_stored":41683036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19037,"internal_metadata":115586467},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":412,"seq":1769526526046,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396960,"kb_used_data":48464,"kb_used_omap":28,"kb_used_meta":348451,"kb_avail":93348192,"statfs":{"total":95995035648,"available":95588548608,"internally_reserved":0,"allocated":49627136,"data_stored":73036018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28947,"internal_metadata":356814573},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":378,"seq":1623497637989,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556156,"kb_used_data":76200,"kb_used_omap":33,"kb_used_meta":479902,"kb_avail":93188996,"statfs":{"total":95995035648,"available":95425531904,"internally_reserved":0,"allocated":78028800,"data_stored":125500653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34165,"internal_metadata":491420299},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":346,"seq":1486058684524,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560380,"kb_used_data":72432,"kb_used_omap":30,"kb_used_meta":487905,"kb_avail":93184772,"statfs":{"total":95995035648,"available":95421206528,"internally_reserved":0,"allocated":74170368,"data_stored":106185628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31205,"internal_metadata":499615259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":576,"seq":2473901162556,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":132080,"kb_used_data":38748,"kb_used_omap":27,"kb_used_meta":93284,"kb_avail":93613072,"statfs":{"total":95995035648,"available":95859785728,"internally_reserved":0,"allocated":39677952,"data_stored":47021384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28412,"internal_metadata":95523076},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":543,"seq":2332167241795,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278396,"kb_used_data":37420,"kb_used_omap":25,"kb_used_meta":240934,"kb_avail":93466756,"statfs":{"total":95995035648,"available":95709958144,"internally_reserved":0,"allocated":38318080,"data_stored":43087442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26328,"internal_metadata":246716712},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":508,"seq":2181843386442,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":426996,"kb_used_data":61992,"kb_used_omap":39,"kb_used_meta":364952,"kb_avail":93318156,"statfs":{"total":95995035648,"available":95557791744,"internally_reserved":0,"allocated":63479808,"data_stored":95818567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40576,"internal_metadata":373711232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":474,"seq":2035814498385,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378540,"kb_used_data":65820,"kb_used_omap":38,"kb_used_meta":312665,"kb_avail":93366612,"statfs":{"total":95995035648,"available":95607410688,"internally_reserved":0,"allocated":67399680,"data_stored":105787779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39328,"internal_metadata":320169568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6751,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4565,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6134,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6698,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6465,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5898,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5161,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5804,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":60731392,"data_stored":98066432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65196032,"data_stored":117972992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":36794368,"data_stored":65507328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":19636224,"data_stored":34177024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":54550528,"data_stored":98258944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":50634752,"data_stored":88285184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24883200,"data_stored":34963456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":26267648,"data_stored":38912000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:32:54.079 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-13T14:32:54.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-13T14:32:54.337 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[97022]: pgmap v886: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[97022]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[97022]: osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[99461]: pgmap v886: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[99461]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:54.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:54 smithi026 ceph-mon[99461]: osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:32:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:54 smithi145 ceph-mon[89145]: pgmap v886: 121 pgs: 121 active+clean; 222 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:32:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:54 smithi145 ceph-mon[89145]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:54.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:54 smithi145 ceph-mon[89145]: osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:32:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:56 smithi145 ceph-mon[89145]: pgmap v888: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:56.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:56 smithi026 ceph-mon[97022]: pgmap v888: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:56 smithi026 ceph-mon[99461]: pgmap v888: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:57.187 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' removed 2024-09-13T14:32:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:57 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1703721968' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:57 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:57 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1703721968' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:57 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:57 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1703721968' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:57 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:32:57.914 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T14:32:57.914 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-13T14:32:57.914 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-13T14:32:57.914 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-13T14:32:57.914 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-13T14:32:57.915 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T14:32:57.916 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:32:57.916 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:32:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.936 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:32:57.936 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:32:57.995 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:32:57.995 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:32:58.054 INFO:tasks.workunit:timeout=3h 2024-09-13T14:32:58.054 INFO:tasks.workunit:cleanup=True 2024-09-13T14:32:58.054 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:32:58.115 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:32:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:58 smithi145 ceph-mon[89145]: pgmap v889: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:58 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:32:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:32:58 smithi145 ceph-mon[89145]: osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:32:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[97022]: pgmap v889: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:32:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[97022]: osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:32:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[99461]: pgmap v889: 121 pgs: 121 active+clean; 221 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:32:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:32:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:32:58 smithi026 ceph-mon[99461]: osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:33:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:00 smithi145 ceph-mon[89145]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:00.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:00 smithi026 ceph-mon[99461]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:00.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:00 smithi026 ceph-mon[97022]: pgmap v891: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:01.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:01 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:01.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:01 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:01 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:01.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:01] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.51.0" 2024-09-13T14:33:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:02 smithi145 ceph-mon[89145]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:33:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[99461]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:33:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[97022]: pgmap v892: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:33:03.210 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:33:03.266 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:33:03.268 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:33:03.269 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:33:03.319 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:33:03.369 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:33:03.410 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:33:03.412 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:33:03.412 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:33:03.453 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:33:03.456 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-09-13T14:33:03.456 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:33:03.513 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-13T14:33:03.514 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-13T14:33:03.515 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-13T14:33:03.581 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:33:03.581 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-13T14:33:03.581 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:33:03.581 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-13T14:33:03.581 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-13T14:33:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:04 smithi145 ceph-mon[89145]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:04 smithi026 ceph-mon[97022]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:04.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:04 smithi026 ceph-mon[99461]: pgmap v893: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:05.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:05 smithi026 ceph-mon[97022]: osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:33:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:05 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:05 smithi026 ceph-mon[99461]: osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:33:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:05 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:05 smithi145 ceph-mon[89145]: osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:33:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:05 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:06.215 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2634 ms) 2024-09-13T14:33:06.215 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-13T14:33:06.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:06 smithi145 ceph-mon[89145]: pgmap v895: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:06.607 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:06 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:06.607 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:06 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:06.607 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:06 smithi145 ceph-mon[89145]: osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:33:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[97022]: pgmap v895: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[97022]: osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[99461]: pgmap v895: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2322191210' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:06 smithi026 ceph-mon[99461]: osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:33:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:07 smithi026 ceph-mon[97022]: osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:33:07.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:07 smithi026 ceph-mon[99461]: osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:33:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:07 smithi145 ceph-mon[89145]: osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:33:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[97022]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[97022]: osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:33:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[99461]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[99461]: osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:33:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:08 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:08 smithi145 ceph-mon[89145]: pgmap v898: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:08 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:08 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:08 smithi145 ceph-mon[89145]: osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:33:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:08 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:09.239 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3024 ms) 2024-09-13T14:33:09.240 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-13T14:33:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:09 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:09.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:09 smithi026 ceph-mon[97022]: osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:33:09.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:09 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:09.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:09 smithi026 ceph-mon[99461]: osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:33:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:09 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2540400194' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:09 smithi145 ceph-mon[89145]: osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:33:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:10 smithi026 ceph-mon[97022]: pgmap v901: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:10 smithi026 ceph-mon[97022]: osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:33:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:10 smithi026 ceph-mon[99461]: pgmap v901: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:10.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:10 smithi026 ceph-mon[99461]: osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:33:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:10 smithi145 ceph-mon[89145]: pgmap v901: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:10 smithi145 ceph-mon[89145]: osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:33:10.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[97022]: osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:33:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:11.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:11] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.51.0" 2024-09-13T14:33:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[99461]: osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:33:11.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:11.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:11 smithi145 ceph-mon[89145]: osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:33:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:11 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:12.258 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.Reserve (3019 ms) 2024-09-13T14:33:12.258 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-13T14:33:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[97022]: pgmap v904: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:12.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[97022]: osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:33:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[99461]: pgmap v904: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:12.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:12 smithi026 ceph-mon[99461]: osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:33:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:12 smithi145 ceph-mon[89145]: pgmap v904: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:12 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:12 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4269293053' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:12 smithi145 ceph-mon[89145]: osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:33:13.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:13 smithi026 ceph-mon[97022]: osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:33:13.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:13 smithi026 ceph-mon[99461]: osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:33:13.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:13 smithi145 ceph-mon[89145]: osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2854811537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:14.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2854811537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:14.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:14 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: pgmap v907: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:33:14.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2854811537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:14.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:14 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:15.277 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3019 ms) 2024-09-13T14:33:15.277 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-13T14:33:15.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:15 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:15.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:15 smithi026 ceph-mon[99461]: osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:33:15.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:15 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:15.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:15 smithi026 ceph-mon[97022]: osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:33:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:15 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:15 smithi145 ceph-mon[89145]: osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:33:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:16 smithi026 ceph-mon[97022]: pgmap v910: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:16 smithi026 ceph-mon[97022]: osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:33:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:16 smithi026 ceph-mon[99461]: pgmap v910: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:16 smithi026 ceph-mon[99461]: osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:33:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:16 smithi145 ceph-mon[89145]: pgmap v910: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:16 smithi145 ceph-mon[89145]: osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:33:17.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:17 smithi026 ceph-mon[97022]: osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:33:17.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:17 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:17.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:17 smithi026 ceph-mon[99461]: osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:33:17.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:17 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:17.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:17 smithi145 ceph-mon[89145]: osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:33:17.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:17 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:18.290 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.Commit (3013 ms) 2024-09-13T14:33:18.290 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-13T14:33:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[97022]: pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[97022]: osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[99461]: pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:18.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:18 smithi026 ceph-mon[99461]: osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:33:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:18 smithi145 ceph-mon[89145]: pgmap v913: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:18 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:18 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3712678592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:18 smithi145 ceph-mon[89145]: osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:33:19.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:19 smithi026 ceph-mon[97022]: osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:33:19.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:19 smithi026 ceph-mon[99461]: osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:33:19.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:19 smithi145 ceph-mon[89145]: osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:33:20.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[97022]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[97022]: osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[99461]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[99461]: osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:33:20.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:20 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-mon[89145]: pgmap v916: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-mon[89145]: osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:33:20.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:20.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:21.316 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.Abort (3026 ms) 2024-09-13T14:33:21.316 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-13T14:33:21.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:21 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:21 smithi026 ceph-mon[97022]: osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:33:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:21 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:21 smithi026 ceph-mon[99461]: osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:33:21.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:21] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.51.0" 2024-09-13T14:33:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:21 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1826148456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:21 smithi145 ceph-mon[89145]: osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:33:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:22 smithi026 ceph-mon[97022]: pgmap v919: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:22.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:22 smithi026 ceph-mon[97022]: osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:33:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:22 smithi026 ceph-mon[99461]: pgmap v919: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:22 smithi026 ceph-mon[99461]: osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:33:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:22 smithi145 ceph-mon[89145]: pgmap v919: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:22 smithi145 ceph-mon[89145]: osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:33:23.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[99461]: osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:33:23.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3770274139' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:23.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[97022]: osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:33:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3770274139' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:23.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:23 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:23 smithi145 ceph-mon[89145]: osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:33:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:23 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3770274139' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:23.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:23 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:24.332 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.ReserveError (3016 ms) 2024-09-13T14:33:24.333 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[97022]: pgmap v922: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[97022]: osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[99461]: pgmap v922: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:24.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:24.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:24 smithi026 ceph-mon[99461]: osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:33:24.711 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:24 smithi145 ceph-mon[89145]: pgmap v922: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:24.711 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:24 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:24.711 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:24 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:24.711 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:24 smithi145 ceph-mon[89145]: osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:33:25.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:25 smithi026 ceph-mon[97022]: osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:33:25.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:25 smithi026 ceph-mon[99461]: osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:33:25.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:25 smithi145 ceph-mon[89145]: osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:26.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:27.360 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.CommitError (3028 ms) 2024-09-13T14:33:27.360 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-13T14:33:27.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:27 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:27.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:27 smithi026 ceph-mon[99461]: osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:33:27.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:27 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:27.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:27 smithi026 ceph-mon[97022]: osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:33:27.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:27 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/175602264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:27.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:27 smithi145 ceph-mon[89145]: osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:33:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:28 smithi026 ceph-mon[97022]: pgmap v928: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:28 smithi026 ceph-mon[97022]: osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:33:28.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:28 smithi026 ceph-mon[99461]: pgmap v928: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:28 smithi026 ceph-mon[99461]: osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:33:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:28 smithi145 ceph-mon[89145]: pgmap v928: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:28 smithi145 ceph-mon[89145]: osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:33:29.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[99461]: osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:33:29.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3753832734' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:29.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[97022]: osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:33:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3753832734' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:29 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:29 smithi145 ceph-mon[89145]: osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:33:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:29 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3753832734' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:29 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:30.381 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.AbortError (3021 ms) 2024-09-13T14:33:30.381 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-13T14:33:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[97022]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[97022]: osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[99461]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:30 smithi026 ceph-mon[99461]: osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:33:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:30 smithi145 ceph-mon[89145]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:30 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:30 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:30 smithi145 ceph-mon[89145]: osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:33:30.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:31 smithi026 ceph-mon[97022]: osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:33:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:31 smithi026 ceph-mon[99461]: osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:33:31.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:31] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-13T14:33:31.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:31 smithi145 ceph-mon[89145]: osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:33:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[99461]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[99461]: osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[97022]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[97022]: osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:33:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:32 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:32 smithi145 ceph-mon[89145]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:32 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:32 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:32 smithi145 ceph-mon[89145]: osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:33:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:32 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:33.405 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3024 ms) 2024-09-13T14:33:33.405 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-13T14:33:33.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:33 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:33.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:33 smithi026 ceph-mon[99461]: osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:33:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:33 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:33 smithi026 ceph-mon[97022]: osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:33:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:33 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/637195600' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:33.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:33 smithi145 ceph-mon[89145]: osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:33:34.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:34 smithi026 ceph-mon[99461]: pgmap v937: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:34 smithi026 ceph-mon[99461]: osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:33:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:34 smithi026 ceph-mon[97022]: pgmap v937: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:34.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:34 smithi026 ceph-mon[97022]: osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:33:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:34 smithi145 ceph-mon[89145]: pgmap v937: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:34 smithi145 ceph-mon[89145]: osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:33:35.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:35 smithi026 ceph-mon[99461]: osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:33:35.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:35 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:35 smithi026 ceph-mon[97022]: osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:33:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:35 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:35 smithi145 ceph-mon[89145]: osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:33:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:35 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:36.425 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3020 ms) 2024-09-13T14:33:36.425 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-13T14:33:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[97022]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[97022]: osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[99461]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:36 smithi026 ceph-mon[99461]: osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:33:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:36 smithi145 ceph-mon[89145]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:36 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:36 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1943201914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:36.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:36 smithi145 ceph-mon[89145]: osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:33:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:37 smithi145 ceph-mon[89145]: osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:33:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:37 smithi026 ceph-mon[97022]: osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:33:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:37 smithi026 ceph-mon[99461]: osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:33:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:33:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1452214356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:38.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:38 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1452214356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:38.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:38.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:38.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:33:38.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1452214356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:38.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:38 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:39.448 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3023 ms) 2024-09-13T14:33:39.449 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-13T14:33:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:39 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:39 smithi145 ceph-mon[89145]: osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:33:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:39 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:39 smithi026 ceph-mon[97022]: osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:33:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:39 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:39 smithi026 ceph-mon[99461]: osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:33:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:40 smithi145 ceph-mon[89145]: pgmap v946: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:40 smithi145 ceph-mon[89145]: osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:33:40.706 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:40 smithi026 ceph-mon[97022]: pgmap v946: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:40.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:40 smithi026 ceph-mon[97022]: osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:33:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:40 smithi026 ceph-mon[99461]: pgmap v946: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:40 smithi026 ceph-mon[99461]: osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:33:41.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:41] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-13T14:33:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[97022]: osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3380439632' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[99461]: osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3380439632' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:41 smithi145 ceph-mon[89145]: osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:33:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:41 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3380439632' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:41 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.466 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3018 ms) 2024-09-13T14:33:42.466 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-13T14:33:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:42 smithi145 ceph-mon[89145]: pgmap v949: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:42 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:42 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:42 smithi145 ceph-mon[89145]: osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:33:42.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[99461]: pgmap v949: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:42.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:42.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:42.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[99461]: osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:33:42.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[97022]: pgmap v949: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:42.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:42.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:42.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:42 smithi026 ceph-mon[97022]: osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:33:43.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:43 smithi026 ceph-mon[97022]: osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:33:43.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:43 smithi026 ceph-mon[99461]: osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:33:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:43 smithi145 ceph-mon[89145]: osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/138206263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:44.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/138206263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:44.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:44 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/138206263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:44.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:44 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:45.488 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3022 ms) 2024-09-13T14:33:45.488 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-13T14:33:45.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:45 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:45.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:45 smithi026 ceph-mon[97022]: osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:33:45.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:45 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:45.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:45 smithi026 ceph-mon[99461]: osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:33:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:45 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:45 smithi145 ceph-mon[89145]: osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:33:46.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:46 smithi026 ceph-mon[99461]: pgmap v955: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:46.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:46 smithi026 ceph-mon[99461]: osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:33:46.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:46 smithi026 ceph-mon[97022]: pgmap v955: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:46.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:46 smithi026 ceph-mon[97022]: osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:33:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:46 smithi145 ceph-mon[89145]: pgmap v955: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:46.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:46 smithi145 ceph-mon[89145]: osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:33:47.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/522255122' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: pgmap v958: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[97022]: osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/522255122' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: pgmap v958: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:47.917 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:47 smithi026 ceph-mon[99461]: osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/522255122' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: pgmap v958: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:47.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:47 smithi145 ceph-mon[89145]: osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:33:48.492 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3004 ms) 2024-09-13T14:33:48.492 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-13T14:33:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:48 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:48.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:48 smithi026 ceph-mon[97022]: osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:33:48.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:48 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:48.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:48 smithi026 ceph-mon[99461]: osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:33:48.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:48 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:48.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:48 smithi145 ceph-mon[89145]: osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:33:49.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:49 smithi026 ceph-mon[97022]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:49.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:49 smithi026 ceph-mon[99461]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:49.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:49 smithi145 ceph-mon[89145]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:50.525 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:33:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:33:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[97022]: osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[99461]: osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:33:50.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:50 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:50 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:50 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:50 smithi145 ceph-mon[89145]: osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:33:50.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:50 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:51.537 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.Cleanup (3045 ms) 2024-09-13T14:33:51.538 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-13T14:33:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[97022]: osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:33:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[97022]: pgmap v964: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:51.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:33:51] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-13T14:33:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[99461]: osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:33:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:51 smithi026 ceph-mon[99461]: pgmap v964: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:51 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1555453034' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:51 smithi145 ceph-mon[89145]: osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:33:51.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:51 smithi145 ceph-mon[89145]: pgmap v964: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:52 smithi026 ceph-mon[97022]: osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:33:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:52 smithi026 ceph-mon[99461]: osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:33:52.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:52 smithi145 ceph-mon[89145]: osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:33:53.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[99461]: osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:33:53.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:53.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[99461]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:53.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[97022]: osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:33:53.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:53.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:53 smithi026 ceph-mon[97022]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:53 smithi145 ceph-mon[89145]: osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:33:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:53 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:53.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:53 smithi145 ceph-mon[89145]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:54.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[99461]: osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:33:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:54 smithi026 ceph-mon[97022]: osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:33:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:54 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:33:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:54 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2805808378' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:54.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:54 smithi145 ceph-mon[89145]: osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:33:55.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:55 smithi026 ceph-mon[99461]: osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:33:55.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:55 smithi026 ceph-mon[99461]: pgmap v970: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:55.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:55 smithi026 ceph-mon[97022]: osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:33:55.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:55 smithi026 ceph-mon[97022]: pgmap v970: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:55.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:55 smithi145 ceph-mon[89145]: osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:33:55.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:55 smithi145 ceph-mon[89145]: pgmap v970: 137 pgs: 3 creating+peering, 24 unknown, 110 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:33:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:33:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:56 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:33:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:56 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:33:56.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:57.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:33:57 smithi026 ceph-mon[99461]: pgmap v971: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.7 MiB/s wr, 6.47k op/s 2024-09-13T14:33:57.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:33:57 smithi026 ceph-mon[97022]: pgmap v971: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.7 MiB/s wr, 6.47k op/s 2024-09-13T14:33:57.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:33:57 smithi145 ceph-mon[89145]: pgmap v971: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.7 MiB/s wr, 6.47k op/s 2024-09-13T14:34:00.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:00 smithi026 ceph-mon[99461]: pgmap v972: 137 pgs: 137 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 5.45k op/s 2024-09-13T14:34:00.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:00 smithi026 ceph-mon[97022]: pgmap v972: 137 pgs: 137 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 5.45k op/s 2024-09-13T14:34:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:00 smithi145 ceph-mon[89145]: pgmap v972: 137 pgs: 137 active+clean; 1.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 5.45k op/s 2024-09-13T14:34:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:01] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-09-13T14:34:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[97022]: pgmap v973: 137 pgs: 137 active+clean; 1.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 4.31k op/s 2024-09-13T14:34:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[99461]: pgmap v973: 137 pgs: 137 active+clean; 1.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 4.31k op/s 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:34:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:02 smithi145 ceph-mon[89145]: pgmap v973: 137 pgs: 137 active+clean; 1.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 4.31k op/s 2024-09-13T14:34:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:34:04.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:04 smithi026 ceph-mon[99461]: pgmap v974: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 7.5 MiB/s wr, 8.47k op/s 2024-09-13T14:34:04.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:04 smithi026 ceph-mon[97022]: pgmap v974: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 7.5 MiB/s wr, 8.47k op/s 2024-09-13T14:34:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:04 smithi145 ceph-mon[89145]: pgmap v974: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 7.5 MiB/s wr, 8.47k op/s 2024-09-13T14:34:06.092 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.MultiProducer (14553 ms) 2024-09-13T14:34:06.092 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-13T14:34:06.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:06 smithi026 ceph-mon[99461]: pgmap v975: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 6.8 MiB/s wr, 7.65k op/s 2024-09-13T14:34:06.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:06 smithi026 ceph-mon[97022]: pgmap v975: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 6.8 MiB/s wr, 7.65k op/s 2024-09-13T14:34:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:06 smithi145 ceph-mon[89145]: pgmap v975: 137 pgs: 137 active+clean; 2.2 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 6.8 MiB/s wr, 7.65k op/s 2024-09-13T14:34:07.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:07 smithi026 ceph-mon[97022]: osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:34:07.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:07 smithi026 ceph-mon[99461]: osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:34:07.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:07 smithi145 ceph-mon[89145]: osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:34:08.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[99461]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:08.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[99461]: osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:34:08.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:08.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[97022]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:08.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[97022]: osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:34:08.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:08 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:08 smithi145 ceph-mon[89145]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:08 smithi145 ceph-mon[89145]: osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:34:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:08 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:09.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:09 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:09.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:09 smithi026 ceph-mon[99461]: osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:34:09.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:09 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:09.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:09 smithi026 ceph-mon[97022]: osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:34:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:09 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2243130283' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:09.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:09 smithi145 ceph-mon[89145]: osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:34:10.119 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4029 ms) 2024-09-13T14:34:10.119 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-13T14:34:10.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:10 smithi026 ceph-mon[97022]: pgmap v980: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:10.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:10 smithi026 ceph-mon[97022]: osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:34:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:10 smithi026 ceph-mon[99461]: pgmap v980: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:10.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:10 smithi026 ceph-mon[99461]: osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:34:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:10 smithi145 ceph-mon[89145]: pgmap v980: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:10 smithi145 ceph-mon[89145]: osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:34:10.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:11.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:11 smithi026 ceph-mon[97022]: osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:34:11.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:11 smithi026 ceph-mon[99461]: osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:34:11.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:11] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.51.0" 2024-09-13T14:34:11.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:11 smithi145 ceph-mon[89145]: osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:34:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[97022]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[97022]: osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:34:12.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:12.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[99461]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[99461]: osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:34:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:12.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:12 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:12 smithi145 ceph-mon[89145]: pgmap v983: 105 pgs: 105 active+clean; 583 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:12 smithi145 ceph-mon[89145]: osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:34:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:12 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:12 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:13 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:13.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:13 smithi026 ceph-mon[97022]: osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:34:13.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:13 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:13.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:13 smithi026 ceph-mon[99461]: osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:34:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:13 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2481762386' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:13.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:13 smithi145 ceph-mon[89145]: osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:34:14.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:14 smithi026 ceph-mon[97022]: pgmap v986: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:14.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:14 smithi026 ceph-mon[97022]: osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:34:14.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:14 smithi026 ceph-mon[99461]: pgmap v986: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:14.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:14 smithi026 ceph-mon[99461]: osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:34:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:14 smithi145 ceph-mon[89145]: pgmap v986: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:14.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:14 smithi145 ceph-mon[89145]: osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:34:16.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:16 smithi026 ceph-mon[99461]: pgmap v988: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 2.15k op/s 2024-09-13T14:34:16.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:16 smithi026 ceph-mon[97022]: pgmap v988: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 2.15k op/s 2024-09-13T14:34:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:16 smithi145 ceph-mon[89145]: pgmap v988: 137 pgs: 3 creating+peering, 19 unknown, 115 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 2.15k op/s 2024-09-13T14:34:18.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:18 smithi026 ceph-mon[97022]: pgmap v989: 137 pgs: 3 creating+peering, 134 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 1.70k op/s 2024-09-13T14:34:18.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:18 smithi026 ceph-mon[99461]: pgmap v989: 137 pgs: 3 creating+peering, 134 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 1.70k op/s 2024-09-13T14:34:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:18 smithi145 ceph-mon[89145]: pgmap v989: 137 pgs: 3 creating+peering, 134 active+clean; 806 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 1.70k op/s 2024-09-13T14:34:20.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:20 smithi026 ceph-mon[99461]: pgmap v990: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.8 MiB/s wr, 6.82k op/s 2024-09-13T14:34:20.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:20 smithi026 ceph-mon[97022]: pgmap v990: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.8 MiB/s wr, 6.82k op/s 2024-09-13T14:34:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:20 smithi145 ceph-mon[89145]: pgmap v990: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 59 MiB/s rd, 5.8 MiB/s wr, 6.82k op/s 2024-09-13T14:34:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:21.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:21] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.51.0" 2024-09-13T14:34:22.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:22 smithi026 ceph-mon[99461]: pgmap v991: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 5.1 MiB/s wr, 6.04k op/s 2024-09-13T14:34:22.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:22 smithi026 ceph-mon[97022]: pgmap v991: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 5.1 MiB/s wr, 6.04k op/s 2024-09-13T14:34:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:22 smithi145 ceph-mon[89145]: pgmap v991: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 5.1 MiB/s wr, 6.04k op/s 2024-09-13T14:34:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:24 smithi026 ceph-mon[99461]: pgmap v992: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.28k op/s 2024-09-13T14:34:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:24 smithi026 ceph-mon[97022]: pgmap v992: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.28k op/s 2024-09-13T14:34:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:24 smithi145 ceph-mon[89145]: pgmap v992: 137 pgs: 137 active+clean; 1.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.28k op/s 2024-09-13T14:34:26.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:26 smithi026 ceph-mon[99461]: pgmap v993: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 4.5 MiB/s wr, 5.43k op/s 2024-09-13T14:34:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:26 smithi026 ceph-mon[97022]: pgmap v993: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 4.5 MiB/s wr, 5.43k op/s 2024-09-13T14:34:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:26 smithi145 ceph-mon[89145]: pgmap v993: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 4.5 MiB/s wr, 5.43k op/s 2024-09-13T14:34:27.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:27.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:28.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:28 smithi026 ceph-mon[99461]: pgmap v994: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.31k op/s 2024-09-13T14:34:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:28 smithi026 ceph-mon[97022]: pgmap v994: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.31k op/s 2024-09-13T14:34:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:28 smithi145 ceph-mon[89145]: pgmap v994: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.31k op/s 2024-09-13T14:34:30.167 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (20047 ms) 2024-09-13T14:34:30.167 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-13T14:34:30.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:30 smithi145 ceph-mon[89145]: pgmap v995: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 103 MiB/s rd, 5.5 MiB/s wr, 6.79k op/s 2024-09-13T14:34:30.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:30 smithi026 ceph-mon[97022]: pgmap v995: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 103 MiB/s rd, 5.5 MiB/s wr, 6.79k op/s 2024-09-13T14:34:30.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:30 smithi026 ceph-mon[99461]: pgmap v995: 137 pgs: 137 active+clean; 1.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 103 MiB/s rd, 5.5 MiB/s wr, 6.79k op/s 2024-09-13T14:34:31.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:31 smithi145 ceph-mon[89145]: osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:34:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:31 smithi026 ceph-mon[97022]: osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:34:31.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:31 smithi026 ceph-mon[99461]: osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:34:31.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:31] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.51.0" 2024-09-13T14:34:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:32 smithi145 ceph-mon[89145]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:32 smithi145 ceph-mon[89145]: osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:34:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:32 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1358909352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:32.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:32 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[99461]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[99461]: osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:34:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1358909352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:32.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[97022]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[97022]: osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:34:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1358909352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:32.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:32 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:33 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:33.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:33 smithi145 ceph-mon[89145]: osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:34:33.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:33 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:33.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:33 smithi026 ceph-mon[99461]: osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:34:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:33 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:33.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:33 smithi026 ceph-mon[97022]: osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:34:34.197 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4031 ms) 2024-09-13T14:34:34.197 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-13T14:34:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:34 smithi145 ceph-mon[89145]: pgmap v1000: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:34.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:34 smithi145 ceph-mon[89145]: osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:34:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:34 smithi026 ceph-mon[97022]: pgmap v1000: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:34 smithi026 ceph-mon[97022]: osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:34:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:34 smithi026 ceph-mon[99461]: pgmap v1000: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:34.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:34 smithi026 ceph-mon[99461]: osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:34:35.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:35 smithi145 ceph-mon[89145]: osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:34:35.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:35 smithi026 ceph-mon[99461]: osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:34:35.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:35 smithi026 ceph-mon[97022]: osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:34:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[97022]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[97022]: osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:34:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[99461]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[99461]: osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:34:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:36 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:36 smithi145 ceph-mon[89145]: pgmap v1003: 105 pgs: 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:36 smithi145 ceph-mon[89145]: osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:34:36.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:36 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:37 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:37.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:37 smithi026 ceph-mon[99461]: osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:34:37.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:37 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:37.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:37 smithi026 ceph-mon[97022]: osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:34:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:37 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1286345861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:37 smithi145 ceph-mon[89145]: osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:34:38.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:38 smithi026 ceph-mon[99461]: pgmap v1006: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:38.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:38 smithi026 ceph-mon[99461]: osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:34:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:38 smithi026 ceph-mon[97022]: pgmap v1006: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:38 smithi026 ceph-mon[97022]: osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:34:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:38 smithi145 ceph-mon[89145]: pgmap v1006: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:38 smithi145 ceph-mon[89145]: osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:34:40.229 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6032 ms) 2024-09-13T14:34:40.230 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-13T14:34:40.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:40 smithi026 ceph-mon[97022]: pgmap v1008: 137 pgs: 12 creating+peering, 14 unknown, 111 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:40.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:40 smithi026 ceph-mon[99461]: pgmap v1008: 137 pgs: 12 creating+peering, 14 unknown, 111 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:40 smithi145 ceph-mon[89145]: pgmap v1008: 137 pgs: 12 creating+peering, 14 unknown, 111 active+clean; 583 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:40.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:41 smithi026 ceph-mon[97022]: osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:34:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:41 smithi026 ceph-mon[99461]: osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:34:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:41] "GET /metrics HTTP/1.1" 200 33936 "" "Prometheus/2.51.0" 2024-09-13T14:34:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:41 smithi145 ceph-mon[89145]: osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:34:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[97022]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[97022]: osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:34:42.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[99461]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[99461]: osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:34:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:42 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:42 smithi145 ceph-mon[89145]: pgmap v1010: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:42 smithi145 ceph-mon[89145]: osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:34:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:42 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:43.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:43 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:43.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:43 smithi026 ceph-mon[99461]: osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:34:43.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:43 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:43.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:43 smithi026 ceph-mon[97022]: osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:34:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:43 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1871478202' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-131716-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:43 smithi145 ceph-mon[89145]: osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:34:44.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:44 smithi026 ceph-mon[99461]: pgmap v1013: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:44.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:44 smithi026 ceph-mon[99461]: osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:34:44.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:44 smithi026 ceph-mon[97022]: pgmap v1013: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:44.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:44 smithi026 ceph-mon[97022]: osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:34:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:44 smithi145 ceph-mon[89145]: pgmap v1013: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:44 smithi145 ceph-mon[89145]: osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:34:45.259 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5030 ms) 2024-09-13T14:34:45.259 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 22 tests from TestCls2PCQueue (101678 ms total) 2024-09-13T14:34:45.260 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:34:45.260 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:34:45.260 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 22 tests from 1 test suite ran. (101678 ms total) 2024-09-13T14:34:45.260 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 22 tests. 2024-09-13T14:34:45.262 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:34:45.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:34:45.291 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-13T14:34:45.291 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-13T14:34:45.363 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:34:45.363 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-13T14:34:45.363 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:34:45.363 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 5 tests from cls_cas 2024-09-13T14:34:45.363 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_cas.get_put 2024-09-13T14:34:46.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:46 smithi026 ceph-mon[99461]: pgmap v1015: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:46.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:46 smithi026 ceph-mon[99461]: osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:34:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:46 smithi026 ceph-mon[97022]: pgmap v1015: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:46 smithi026 ceph-mon[97022]: osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:34:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:46 smithi145 ceph-mon[89145]: pgmap v1015: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:46 smithi145 ceph-mon[89145]: osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:34:47.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[99461]: osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:34:47.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4157529875' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:47.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:47.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[97022]: osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:34:47.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4157529875' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:47.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:47 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:47 smithi145 ceph-mon[89145]: osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:34:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:47 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4157529875' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:47 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:48.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_cas.get_put (2916 ms) 2024-09-13T14:34:48.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_cas.wrong_put 2024-09-13T14:34:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[97022]: pgmap v1018: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:48.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[97022]: osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[99461]: pgmap v1018: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:48 smithi026 ceph-mon[99461]: osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:34:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:48 smithi145 ceph-mon[89145]: pgmap v1018: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:48 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:48 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:48 smithi145 ceph-mon[89145]: osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:34:49.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:49 smithi026 ceph-mon[97022]: osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:34:49.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:49 smithi026 ceph-mon[99461]: osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:34:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:49 smithi145 ceph-mon[89145]: osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:34:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[97022]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[97022]: osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[99461]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[99461]: osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:34:50.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:50 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:34:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-mon[89145]: pgmap v1021: 105 pgs: 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:34:50.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-mon[89145]: osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:34:50.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:50 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:51.305 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_cas.wrong_put (3026 ms) 2024-09-13T14:34:51.305 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_cas.dup_get 2024-09-13T14:34:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:51 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:51.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:51 smithi026 ceph-mon[97022]: osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:34:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:51 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:51 smithi026 ceph-mon[99461]: osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:34:51.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:34:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:34:51] "GET /metrics HTTP/1.1" 200 33936 "" "Prometheus/2.51.0" 2024-09-13T14:34:51.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:51 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/131812977' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:51.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:51 smithi145 ceph-mon[89145]: osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:34:52.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:52 smithi026 ceph-mon[97022]: pgmap v1024: 137 pgs: 16 creating+peering, 13 unknown, 108 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 1 op/s 2024-09-13T14:34:52.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:52 smithi026 ceph-mon[97022]: osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:34:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:52 smithi026 ceph-mon[99461]: pgmap v1024: 137 pgs: 16 creating+peering, 13 unknown, 108 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 1 op/s 2024-09-13T14:34:52.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:52 smithi026 ceph-mon[99461]: osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:34:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:52 smithi145 ceph-mon[89145]: pgmap v1024: 137 pgs: 16 creating+peering, 13 unknown, 108 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 1 op/s 2024-09-13T14:34:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:52 smithi145 ceph-mon[89145]: osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:34:53.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[99461]: osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:34:53.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:53.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:53.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[99461]: osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:34:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[97022]: osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:34:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:53.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:53 smithi026 ceph-mon[97022]: osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:34:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:53 smithi145 ceph-mon[89145]: osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:34:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:53 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:53 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/270528565' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:53 smithi145 ceph-mon[89145]: osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:34:54.308 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_cas.dup_get (3003 ms) 2024-09-13T14:34:54.308 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_cas.dup_put 2024-09-13T14:34:54.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[97022]: pgmap v1027: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:54.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:54.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[97022]: osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:34:54.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[99461]: pgmap v1027: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:54.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:54.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:54 smithi026 ceph-mon[99461]: osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:34:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:54 smithi145 ceph-mon[89145]: pgmap v1027: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:54 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:54 smithi145 ceph-mon[89145]: osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:34:55.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:55.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[97022]: osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[99461]: osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:34:55.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:55 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:55 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:34:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:55 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:34:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:55 smithi145 ceph-mon[89145]: osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:34:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:55 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[97022]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[97022]: osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[99461]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:56.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:56 smithi026 ceph-mon[99461]: osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:34:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:56 smithi145 ceph-mon[89145]: pgmap v1030: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:56 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2218323828' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:34:56.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:56 smithi145 ceph-mon[89145]: osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:34:57.346 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_cas.dup_put (3038 ms) 2024-09-13T14:34:57.347 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-13T14:34:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:58 smithi026 ceph-mon[97022]: pgmap v1033: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:58.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:58 smithi026 ceph-mon[97022]: osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:34:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:58 smithi026 ceph-mon[99461]: pgmap v1033: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:58 smithi026 ceph-mon[99461]: osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:34:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:58 smithi145 ceph-mon[89145]: pgmap v1033: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:34:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:58 smithi145 ceph-mon[89145]: osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:34:59.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:59 smithi026 ceph-mon[99461]: osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:34:59.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:34:59 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:59.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:59 smithi026 ceph-mon[97022]: osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:34:59.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:34:59 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:34:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:59 smithi145 ceph-mon[89145]: osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:34:59.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:34:59 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:00.363 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_cas.get_wrong_data (3016 ms) 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 5 tests from cls_cas (14999 ms total) 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 1 test from chunk_refs_t 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] chunk_refs_t.size 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:1 60 by_object 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:2 103 by_object 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:4 189 by_object 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:8 361 by_object 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:16 112 by_hash(32 bits) 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:32 208 by_hash(32 bits) 2024-09-13T14:35:00.364 INFO:tasks.workunit.client.0.smithi026.stdout:64 400 by_hash(32 bits) 2024-09-13T14:35:00.365 INFO:tasks.workunit.client.0.smithi026.stdout:128 509 by_hash(10 bits) 2024-09-13T14:35:00.365 INFO:tasks.workunit.client.0.smithi026.stdout:256 499 by_hash(7 bits) 2024-09-13T14:35:00.365 INFO:tasks.workunit.client.0.smithi026.stdout:512 462 by_hash(5 bits) 2024-09-13T14:35:00.365 INFO:tasks.workunit.client.0.smithi026.stdout:1024 466 by_hash(5 bits) 2024-09-13T14:35:00.365 INFO:tasks.workunit.client.0.smithi026.stdout:2048 466 by_hash(5 bits) 2024-09-13T14:35:00.366 INFO:tasks.workunit.client.0.smithi026.stdout:4096 470 by_hash(5 bits) 2024-09-13T14:35:00.366 INFO:tasks.workunit.client.0.smithi026.stdout:8192 498 by_hash(4 bits) 2024-09-13T14:35:00.367 INFO:tasks.workunit.client.0.smithi026.stdout:16384 278 by_hash(3 bits) 2024-09-13T14:35:00.370 INFO:tasks.workunit.client.0.smithi026.stdout:32768 337 by_hash(2 bits) 2024-09-13T14:35:00.374 INFO:tasks.workunit.client.0.smithi026.stdout:65536 367 by_hash(2 bits) 2024-09-13T14:35:00.384 INFO:tasks.workunit.client.0.smithi026.stdout:131072 387 by_hash(1 bits) 2024-09-13T14:35:00.399 INFO:tasks.workunit.client.0.smithi026.stdout:262144 331 by_pool 2024-09-13T14:35:00.429 INFO:tasks.workunit.client.0.smithi026.stdout:524288 21 count 2024-09-13T14:35:00.481 INFO:tasks.workunit.client.0.smithi026.stdout:1048576 21 count 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] chunk_refs_t.size (230 ms) 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 1 test from chunk_refs_t (230 ms total) 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 6 tests from 2 test suites ran. (15230 ms total) 2024-09-13T14:35:00.593 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 6 tests. 2024-09-13T14:35:00.595 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:35:00.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:35:00.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[97022]: pgmap v1036: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:00.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:00.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:00.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[97022]: osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:35:00.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[99461]: pgmap v1036: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:00.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:00.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:00.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:00 smithi026 ceph-mon[99461]: osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:35:00.662 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-13T14:35:00.662 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-13T14:35:00.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:00 smithi145 ceph-mon[89145]: pgmap v1036: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:00 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:00 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1576719189' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132382-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:00.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:00 smithi145 ceph-mon[89145]: osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:35:00.730 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:35:00.730 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-13T14:35:00.730 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:35:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:01 smithi026 ceph-mon[97022]: osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:35:01.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:01 smithi026 ceph-mon[99461]: osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:35:01.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:01] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-13T14:35:01.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:01 smithi145 ceph-mon[89145]: osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:35:02.373 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 31 tests from CmpOmap 2024-09-13T14:35:02.373 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-13T14:35:02.392 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (19 ms) 2024-09-13T14:35:02.392 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-13T14:35:02.395 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (3 ms) 2024-09-13T14:35:02.395 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-13T14:35:02.398 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2024-09-13T14:35:02.398 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-13T14:35:02.401 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (3 ms) 2024-09-13T14:35:02.401 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-13T14:35:02.406 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_str (5 ms) 2024-09-13T14:35:02.406 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-13T14:35:02.411 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_u64 (6 ms) 2024-09-13T14:35:02.412 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-13T14:35:02.415 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (3 ms) 2024-09-13T14:35:02.415 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-13T14:35:02.417 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (3 ms) 2024-09-13T14:35:02.418 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-13T14:35:02.420 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2024-09-13T14:35:02.420 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-13T14:35:02.424 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (4 ms) 2024-09-13T14:35:02.424 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-13T14:35:02.424 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-13T14:35:02.425 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-13T14:35:02.425 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (0 ms) 2024-09-13T14:35:02.425 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-13T14:35:02.431 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (6 ms) 2024-09-13T14:35:02.431 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-13T14:35:02.438 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (7 ms) 2024-09-13T14:35:02.438 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-13T14:35:02.443 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (5 ms) 2024-09-13T14:35:02.443 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-13T14:35:02.450 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2024-09-13T14:35:02.450 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-13T14:35:02.457 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_str (7 ms) 2024-09-13T14:35:02.457 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-13T14:35:02.463 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (6 ms) 2024-09-13T14:35:02.463 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-13T14:35:02.465 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (2 ms) 2024-09-13T14:35:02.465 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-13T14:35:02.467 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2024-09-13T14:35:02.467 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-13T14:35:02.468 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2024-09-13T14:35:02.468 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-13T14:35:02.469 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2024-09-13T14:35:02.469 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-13T14:35:02.475 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (6 ms) 2024-09-13T14:35:02.475 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-13T14:35:02.480 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (6 ms) 2024-09-13T14:35:02.481 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-13T14:35:02.487 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (6 ms) 2024-09-13T14:35:02.487 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-13T14:35:02.493 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (7 ms) 2024-09-13T14:35:02.494 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-13T14:35:02.495 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (1 ms) 2024-09-13T14:35:02.495 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-13T14:35:02.497 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2024-09-13T14:35:02.497 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-13T14:35:02.498 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2024-09-13T14:35:02.498 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-13T14:35:02.499 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (1 ms) 2024-09-13T14:35:02.499 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-13T14:35:02.507 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (8 ms) 2024-09-13T14:35:02.507 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 31 tests from CmpOmap (134 ms total) 2024-09-13T14:35:02.507 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:35:02.507 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3042748652' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:02.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3042748652' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:02.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:02 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: pgmap v1039: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3042748652' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:02.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:02.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:02 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:03.381 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 31 tests from 1 test suite ran. (2651 ms total) 2024-09-13T14:35:03.382 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 31 tests. 2024-09-13T14:35:03.385 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:35:03.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:35:03.455 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-13T14:35:03.455 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-13T14:35:03.525 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:35:03.526 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-13T14:35:03.526 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:35:03.526 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 6 tests from ClsHello 2024-09-13T14:35:03.526 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.SayHello 2024-09-13T14:35:03.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:03 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:03.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:03 smithi026 ceph-mon[99461]: osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:35:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:03 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:03 smithi026 ceph-mon[97022]: osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:35:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:03 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132505-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:03.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:03 smithi145 ceph-mon[89145]: osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:35:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:04 smithi026 ceph-mon[97022]: pgmap v1042: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:04 smithi026 ceph-mon[97022]: osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:35:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:04 smithi026 ceph-mon[99461]: pgmap v1042: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:04.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:04 smithi026 ceph-mon[99461]: osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:35:04.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:04 smithi145 ceph-mon[89145]: pgmap v1042: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:04.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:04 smithi145 ceph-mon[89145]: osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:35:05.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:05 smithi026 ceph-mon[97022]: osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:35:05.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:05 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:05 smithi026 ceph-mon[99461]: osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:35:05.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:05 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:05 smithi145 ceph-mon[89145]: osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:35:05.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:05 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:06.401 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.SayHello (2876 ms) 2024-09-13T14:35:06.401 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.RecordHello 2024-09-13T14:35:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[97022]: pgmap v1045: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[97022]: osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[99461]: pgmap v1045: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:06.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:06 smithi026 ceph-mon[99461]: osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:35:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:06 smithi145 ceph-mon[89145]: pgmap v1045: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:06 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:06 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1441989967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:06.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:06 smithi145 ceph-mon[89145]: osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:35:07.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:07 smithi026 ceph-mon[97022]: osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:35:07.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:07 smithi026 ceph-mon[99461]: osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:35:07.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:07 smithi145 ceph-mon[89145]: osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:35:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3133261976' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3133261976' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:08 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: pgmap v1048: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:08.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:35:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3133261976' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:08 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:09.421 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.RecordHello (3020 ms) 2024-09-13T14:35:09.422 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-13T14:35:09.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:09 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:09.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:09 smithi026 ceph-mon[99461]: osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:35:09.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:09 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:09.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:09 smithi026 ceph-mon[97022]: osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:35:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:09 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:09.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:09 smithi145 ceph-mon[89145]: osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:35:10.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:10 smithi026 ceph-mon[99461]: pgmap v1051: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:10 smithi026 ceph-mon[99461]: osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:35:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:10 smithi026 ceph-mon[97022]: pgmap v1051: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:10 smithi026 ceph-mon[97022]: osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:35:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:10 smithi145 ceph-mon[89145]: pgmap v1051: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:10 smithi145 ceph-mon[89145]: osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:35:10.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:11.437 INFO:tasks.workunit.client.0.smithi026.stdout:require_osd_release = squid 2024-09-13T14:35:11.447 INFO:tasks.workunit.client.0.smithi026.stdout:0 2024-09-13T14:35:11.449 INFO:tasks.workunit.client.0.smithi026.stdout:1 2024-09-13T14:35:11.450 INFO:tasks.workunit.client.0.smithi026.stdout:2 2024-09-13T14:35:11.451 INFO:tasks.workunit.client.0.smithi026.stdout:3 2024-09-13T14:35:11.452 INFO:tasks.workunit.client.0.smithi026.stdout:4 2024-09-13T14:35:11.453 INFO:tasks.workunit.client.0.smithi026.stdout:5 2024-09-13T14:35:11.454 INFO:tasks.workunit.client.0.smithi026.stdout:6 2024-09-13T14:35:11.455 INFO:tasks.workunit.client.0.smithi026.stdout:7 2024-09-13T14:35:11.456 INFO:tasks.workunit.client.0.smithi026.stdout:8 2024-09-13T14:35:11.457 INFO:tasks.workunit.client.0.smithi026.stdout:9 2024-09-13T14:35:11.458 INFO:tasks.workunit.client.0.smithi026.stdout:10 2024-09-13T14:35:11.459 INFO:tasks.workunit.client.0.smithi026.stdout:11 2024-09-13T14:35:11.460 INFO:tasks.workunit.client.0.smithi026.stdout:12 2024-09-13T14:35:11.461 INFO:tasks.workunit.client.0.smithi026.stdout:13 2024-09-13T14:35:11.462 INFO:tasks.workunit.client.0.smithi026.stdout:14 2024-09-13T14:35:11.462 INFO:tasks.workunit.client.0.smithi026.stdout:15 2024-09-13T14:35:11.463 INFO:tasks.workunit.client.0.smithi026.stdout:16 2024-09-13T14:35:11.464 INFO:tasks.workunit.client.0.smithi026.stdout:17 2024-09-13T14:35:11.465 INFO:tasks.workunit.client.0.smithi026.stdout:18 2024-09-13T14:35:11.466 INFO:tasks.workunit.client.0.smithi026.stdout:19 2024-09-13T14:35:11.467 INFO:tasks.workunit.client.0.smithi026.stdout:20 2024-09-13T14:35:11.468 INFO:tasks.workunit.client.0.smithi026.stdout:21 2024-09-13T14:35:11.469 INFO:tasks.workunit.client.0.smithi026.stdout:22 2024-09-13T14:35:11.470 INFO:tasks.workunit.client.0.smithi026.stdout:23 2024-09-13T14:35:11.471 INFO:tasks.workunit.client.0.smithi026.stdout:24 2024-09-13T14:35:11.472 INFO:tasks.workunit.client.0.smithi026.stdout:25 2024-09-13T14:35:11.473 INFO:tasks.workunit.client.0.smithi026.stdout:26 2024-09-13T14:35:11.474 INFO:tasks.workunit.client.0.smithi026.stdout:27 2024-09-13T14:35:11.475 INFO:tasks.workunit.client.0.smithi026.stdout:28 2024-09-13T14:35:11.476 INFO:tasks.workunit.client.0.smithi026.stdout:29 2024-09-13T14:35:11.477 INFO:tasks.workunit.client.0.smithi026.stdout:30 2024-09-13T14:35:11.478 INFO:tasks.workunit.client.0.smithi026.stdout:31 2024-09-13T14:35:11.478 INFO:tasks.workunit.client.0.smithi026.stdout:32 2024-09-13T14:35:11.479 INFO:tasks.workunit.client.0.smithi026.stdout:33 2024-09-13T14:35:11.480 INFO:tasks.workunit.client.0.smithi026.stdout:34 2024-09-13T14:35:11.481 INFO:tasks.workunit.client.0.smithi026.stdout:35 2024-09-13T14:35:11.482 INFO:tasks.workunit.client.0.smithi026.stdout:36 2024-09-13T14:35:11.483 INFO:tasks.workunit.client.0.smithi026.stdout:37 2024-09-13T14:35:11.484 INFO:tasks.workunit.client.0.smithi026.stdout:38 2024-09-13T14:35:11.485 INFO:tasks.workunit.client.0.smithi026.stdout:39 2024-09-13T14:35:11.486 INFO:tasks.workunit.client.0.smithi026.stdout:40 2024-09-13T14:35:11.487 INFO:tasks.workunit.client.0.smithi026.stdout:41 2024-09-13T14:35:11.488 INFO:tasks.workunit.client.0.smithi026.stdout:42 2024-09-13T14:35:11.489 INFO:tasks.workunit.client.0.smithi026.stdout:43 2024-09-13T14:35:11.489 INFO:tasks.workunit.client.0.smithi026.stdout:44 2024-09-13T14:35:11.490 INFO:tasks.workunit.client.0.smithi026.stdout:45 2024-09-13T14:35:11.491 INFO:tasks.workunit.client.0.smithi026.stdout:46 2024-09-13T14:35:11.492 INFO:tasks.workunit.client.0.smithi026.stdout:47 2024-09-13T14:35:11.493 INFO:tasks.workunit.client.0.smithi026.stdout:48 2024-09-13T14:35:11.494 INFO:tasks.workunit.client.0.smithi026.stdout:49 2024-09-13T14:35:11.495 INFO:tasks.workunit.client.0.smithi026.stdout:50 2024-09-13T14:35:11.496 INFO:tasks.workunit.client.0.smithi026.stdout:51 2024-09-13T14:35:11.497 INFO:tasks.workunit.client.0.smithi026.stdout:52 2024-09-13T14:35:11.498 INFO:tasks.workunit.client.0.smithi026.stdout:53 2024-09-13T14:35:11.499 INFO:tasks.workunit.client.0.smithi026.stdout:54 2024-09-13T14:35:11.500 INFO:tasks.workunit.client.0.smithi026.stdout:55 2024-09-13T14:35:11.501 INFO:tasks.workunit.client.0.smithi026.stdout:56 2024-09-13T14:35:11.502 INFO:tasks.workunit.client.0.smithi026.stdout:57 2024-09-13T14:35:11.503 INFO:tasks.workunit.client.0.smithi026.stdout:58 2024-09-13T14:35:11.503 INFO:tasks.workunit.client.0.smithi026.stdout:59 2024-09-13T14:35:11.504 INFO:tasks.workunit.client.0.smithi026.stdout:60 2024-09-13T14:35:11.505 INFO:tasks.workunit.client.0.smithi026.stdout:61 2024-09-13T14:35:11.506 INFO:tasks.workunit.client.0.smithi026.stdout:62 2024-09-13T14:35:11.507 INFO:tasks.workunit.client.0.smithi026.stdout:63 2024-09-13T14:35:11.508 INFO:tasks.workunit.client.0.smithi026.stdout:64 2024-09-13T14:35:11.509 INFO:tasks.workunit.client.0.smithi026.stdout:65 2024-09-13T14:35:11.509 INFO:tasks.workunit.client.0.smithi026.stdout:66 2024-09-13T14:35:11.510 INFO:tasks.workunit.client.0.smithi026.stdout:67 2024-09-13T14:35:11.511 INFO:tasks.workunit.client.0.smithi026.stdout:68 2024-09-13T14:35:11.512 INFO:tasks.workunit.client.0.smithi026.stdout:69 2024-09-13T14:35:11.513 INFO:tasks.workunit.client.0.smithi026.stdout:70 2024-09-13T14:35:11.514 INFO:tasks.workunit.client.0.smithi026.stdout:71 2024-09-13T14:35:11.515 INFO:tasks.workunit.client.0.smithi026.stdout:72 2024-09-13T14:35:11.516 INFO:tasks.workunit.client.0.smithi026.stdout:73 2024-09-13T14:35:11.516 INFO:tasks.workunit.client.0.smithi026.stdout:74 2024-09-13T14:35:11.517 INFO:tasks.workunit.client.0.smithi026.stdout:75 2024-09-13T14:35:11.518 INFO:tasks.workunit.client.0.smithi026.stdout:76 2024-09-13T14:35:11.519 INFO:tasks.workunit.client.0.smithi026.stdout:77 2024-09-13T14:35:11.520 INFO:tasks.workunit.client.0.smithi026.stdout:78 2024-09-13T14:35:11.521 INFO:tasks.workunit.client.0.smithi026.stdout:79 2024-09-13T14:35:11.522 INFO:tasks.workunit.client.0.smithi026.stdout:80 2024-09-13T14:35:11.522 INFO:tasks.workunit.client.0.smithi026.stdout:81 2024-09-13T14:35:11.523 INFO:tasks.workunit.client.0.smithi026.stdout:82 2024-09-13T14:35:11.524 INFO:tasks.workunit.client.0.smithi026.stdout:83 2024-09-13T14:35:11.525 INFO:tasks.workunit.client.0.smithi026.stdout:84 2024-09-13T14:35:11.526 INFO:tasks.workunit.client.0.smithi026.stdout:85 2024-09-13T14:35:11.527 INFO:tasks.workunit.client.0.smithi026.stdout:86 2024-09-13T14:35:11.527 INFO:tasks.workunit.client.0.smithi026.stdout:87 2024-09-13T14:35:11.528 INFO:tasks.workunit.client.0.smithi026.stdout:88 2024-09-13T14:35:11.529 INFO:tasks.workunit.client.0.smithi026.stdout:89 2024-09-13T14:35:11.530 INFO:tasks.workunit.client.0.smithi026.stdout:90 2024-09-13T14:35:11.531 INFO:tasks.workunit.client.0.smithi026.stdout:91 2024-09-13T14:35:11.531 INFO:tasks.workunit.client.0.smithi026.stdout:92 2024-09-13T14:35:11.532 INFO:tasks.workunit.client.0.smithi026.stdout:93 2024-09-13T14:35:11.533 INFO:tasks.workunit.client.0.smithi026.stdout:94 2024-09-13T14:35:11.534 INFO:tasks.workunit.client.0.smithi026.stdout:95 2024-09-13T14:35:11.535 INFO:tasks.workunit.client.0.smithi026.stdout:96 2024-09-13T14:35:11.536 INFO:tasks.workunit.client.0.smithi026.stdout:97 2024-09-13T14:35:11.537 INFO:tasks.workunit.client.0.smithi026.stdout:98 2024-09-13T14:35:11.537 INFO:tasks.workunit.client.0.smithi026.stdout:99 2024-09-13T14:35:11.538 INFO:tasks.workunit.client.0.smithi026.stdout:100 2024-09-13T14:35:11.539 INFO:tasks.workunit.client.0.smithi026.stdout:101 2024-09-13T14:35:11.540 INFO:tasks.workunit.client.0.smithi026.stdout:102 2024-09-13T14:35:11.541 INFO:tasks.workunit.client.0.smithi026.stdout:103 2024-09-13T14:35:11.541 INFO:tasks.workunit.client.0.smithi026.stdout:104 2024-09-13T14:35:11.542 INFO:tasks.workunit.client.0.smithi026.stdout:105 2024-09-13T14:35:11.543 INFO:tasks.workunit.client.0.smithi026.stdout:106 2024-09-13T14:35:11.544 INFO:tasks.workunit.client.0.smithi026.stdout:107 2024-09-13T14:35:11.545 INFO:tasks.workunit.client.0.smithi026.stdout:108 2024-09-13T14:35:11.546 INFO:tasks.workunit.client.0.smithi026.stdout:109 2024-09-13T14:35:11.547 INFO:tasks.workunit.client.0.smithi026.stdout:110 2024-09-13T14:35:11.547 INFO:tasks.workunit.client.0.smithi026.stdout:111 2024-09-13T14:35:11.548 INFO:tasks.workunit.client.0.smithi026.stdout:112 2024-09-13T14:35:11.549 INFO:tasks.workunit.client.0.smithi026.stdout:113 2024-09-13T14:35:11.550 INFO:tasks.workunit.client.0.smithi026.stdout:114 2024-09-13T14:35:11.551 INFO:tasks.workunit.client.0.smithi026.stdout:115 2024-09-13T14:35:11.552 INFO:tasks.workunit.client.0.smithi026.stdout:116 2024-09-13T14:35:11.553 INFO:tasks.workunit.client.0.smithi026.stdout:117 2024-09-13T14:35:11.553 INFO:tasks.workunit.client.0.smithi026.stdout:118 2024-09-13T14:35:11.554 INFO:tasks.workunit.client.0.smithi026.stdout:119 2024-09-13T14:35:11.555 INFO:tasks.workunit.client.0.smithi026.stdout:120 2024-09-13T14:35:11.556 INFO:tasks.workunit.client.0.smithi026.stdout:121 2024-09-13T14:35:11.557 INFO:tasks.workunit.client.0.smithi026.stdout:122 2024-09-13T14:35:11.558 INFO:tasks.workunit.client.0.smithi026.stdout:123 2024-09-13T14:35:11.558 INFO:tasks.workunit.client.0.smithi026.stdout:124 2024-09-13T14:35:11.559 INFO:tasks.workunit.client.0.smithi026.stdout:125 2024-09-13T14:35:11.560 INFO:tasks.workunit.client.0.smithi026.stdout:126 2024-09-13T14:35:11.561 INFO:tasks.workunit.client.0.smithi026.stdout:127 2024-09-13T14:35:11.562 INFO:tasks.workunit.client.0.smithi026.stdout:128 2024-09-13T14:35:11.563 INFO:tasks.workunit.client.0.smithi026.stdout:129 2024-09-13T14:35:11.563 INFO:tasks.workunit.client.0.smithi026.stdout:130 2024-09-13T14:35:11.564 INFO:tasks.workunit.client.0.smithi026.stdout:131 2024-09-13T14:35:11.565 INFO:tasks.workunit.client.0.smithi026.stdout:132 2024-09-13T14:35:11.566 INFO:tasks.workunit.client.0.smithi026.stdout:133 2024-09-13T14:35:11.567 INFO:tasks.workunit.client.0.smithi026.stdout:134 2024-09-13T14:35:11.568 INFO:tasks.workunit.client.0.smithi026.stdout:135 2024-09-13T14:35:11.569 INFO:tasks.workunit.client.0.smithi026.stdout:136 2024-09-13T14:35:11.570 INFO:tasks.workunit.client.0.smithi026.stdout:137 2024-09-13T14:35:11.570 INFO:tasks.workunit.client.0.smithi026.stdout:138 2024-09-13T14:35:11.571 INFO:tasks.workunit.client.0.smithi026.stdout:139 2024-09-13T14:35:11.572 INFO:tasks.workunit.client.0.smithi026.stdout:140 2024-09-13T14:35:11.573 INFO:tasks.workunit.client.0.smithi026.stdout:141 2024-09-13T14:35:11.574 INFO:tasks.workunit.client.0.smithi026.stdout:142 2024-09-13T14:35:11.575 INFO:tasks.workunit.client.0.smithi026.stdout:143 2024-09-13T14:35:11.576 INFO:tasks.workunit.client.0.smithi026.stdout:144 2024-09-13T14:35:11.576 INFO:tasks.workunit.client.0.smithi026.stdout:145 2024-09-13T14:35:11.577 INFO:tasks.workunit.client.0.smithi026.stdout:146 2024-09-13T14:35:11.578 INFO:tasks.workunit.client.0.smithi026.stdout:147 2024-09-13T14:35:11.579 INFO:tasks.workunit.client.0.smithi026.stdout:148 2024-09-13T14:35:11.580 INFO:tasks.workunit.client.0.smithi026.stdout:149 2024-09-13T14:35:11.581 INFO:tasks.workunit.client.0.smithi026.stdout:150 2024-09-13T14:35:11.582 INFO:tasks.workunit.client.0.smithi026.stdout:151 2024-09-13T14:35:11.583 INFO:tasks.workunit.client.0.smithi026.stdout:152 2024-09-13T14:35:11.584 INFO:tasks.workunit.client.0.smithi026.stdout:153 2024-09-13T14:35:11.585 INFO:tasks.workunit.client.0.smithi026.stdout:154 2024-09-13T14:35:11.586 INFO:tasks.workunit.client.0.smithi026.stdout:155 2024-09-13T14:35:11.587 INFO:tasks.workunit.client.0.smithi026.stdout:156 2024-09-13T14:35:11.587 INFO:tasks.workunit.client.0.smithi026.stdout:157 2024-09-13T14:35:11.588 INFO:tasks.workunit.client.0.smithi026.stdout:158 2024-09-13T14:35:11.589 INFO:tasks.workunit.client.0.smithi026.stdout:159 2024-09-13T14:35:11.592 INFO:tasks.workunit.client.0.smithi026.stdout:160 2024-09-13T14:35:11.592 INFO:tasks.workunit.client.0.smithi026.stdout:161 2024-09-13T14:35:11.592 INFO:tasks.workunit.client.0.smithi026.stdout:162 2024-09-13T14:35:11.593 INFO:tasks.workunit.client.0.smithi026.stdout:163 2024-09-13T14:35:11.594 INFO:tasks.workunit.client.0.smithi026.stdout:164 2024-09-13T14:35:11.595 INFO:tasks.workunit.client.0.smithi026.stdout:165 2024-09-13T14:35:11.596 INFO:tasks.workunit.client.0.smithi026.stdout:166 2024-09-13T14:35:11.597 INFO:tasks.workunit.client.0.smithi026.stdout:167 2024-09-13T14:35:11.598 INFO:tasks.workunit.client.0.smithi026.stdout:168 2024-09-13T14:35:11.599 INFO:tasks.workunit.client.0.smithi026.stdout:169 2024-09-13T14:35:11.600 INFO:tasks.workunit.client.0.smithi026.stdout:170 2024-09-13T14:35:11.601 INFO:tasks.workunit.client.0.smithi026.stdout:171 2024-09-13T14:35:11.602 INFO:tasks.workunit.client.0.smithi026.stdout:172 2024-09-13T14:35:11.603 INFO:tasks.workunit.client.0.smithi026.stdout:173 2024-09-13T14:35:11.604 INFO:tasks.workunit.client.0.smithi026.stdout:174 2024-09-13T14:35:11.605 INFO:tasks.workunit.client.0.smithi026.stdout:175 2024-09-13T14:35:11.606 INFO:tasks.workunit.client.0.smithi026.stdout:176 2024-09-13T14:35:11.607 INFO:tasks.workunit.client.0.smithi026.stdout:177 2024-09-13T14:35:11.608 INFO:tasks.workunit.client.0.smithi026.stdout:178 2024-09-13T14:35:11.609 INFO:tasks.workunit.client.0.smithi026.stdout:179 2024-09-13T14:35:11.610 INFO:tasks.workunit.client.0.smithi026.stdout:180 2024-09-13T14:35:11.611 INFO:tasks.workunit.client.0.smithi026.stdout:181 2024-09-13T14:35:11.612 INFO:tasks.workunit.client.0.smithi026.stdout:182 2024-09-13T14:35:11.613 INFO:tasks.workunit.client.0.smithi026.stdout:183 2024-09-13T14:35:11.614 INFO:tasks.workunit.client.0.smithi026.stdout:184 2024-09-13T14:35:11.615 INFO:tasks.workunit.client.0.smithi026.stdout:185 2024-09-13T14:35:11.616 INFO:tasks.workunit.client.0.smithi026.stdout:186 2024-09-13T14:35:11.617 INFO:tasks.workunit.client.0.smithi026.stdout:187 2024-09-13T14:35:11.618 INFO:tasks.workunit.client.0.smithi026.stdout:188 2024-09-13T14:35:11.619 INFO:tasks.workunit.client.0.smithi026.stdout:189 2024-09-13T14:35:11.620 INFO:tasks.workunit.client.0.smithi026.stdout:190 2024-09-13T14:35:11.621 INFO:tasks.workunit.client.0.smithi026.stdout:191 2024-09-13T14:35:11.622 INFO:tasks.workunit.client.0.smithi026.stdout:192 2024-09-13T14:35:11.623 INFO:tasks.workunit.client.0.smithi026.stdout:193 2024-09-13T14:35:11.624 INFO:tasks.workunit.client.0.smithi026.stdout:194 2024-09-13T14:35:11.625 INFO:tasks.workunit.client.0.smithi026.stdout:195 2024-09-13T14:35:11.626 INFO:tasks.workunit.client.0.smithi026.stdout:196 2024-09-13T14:35:11.627 INFO:tasks.workunit.client.0.smithi026.stdout:197 2024-09-13T14:35:11.628 INFO:tasks.workunit.client.0.smithi026.stdout:198 2024-09-13T14:35:11.629 INFO:tasks.workunit.client.0.smithi026.stdout:199 2024-09-13T14:35:11.630 INFO:tasks.workunit.client.0.smithi026.stdout:200 2024-09-13T14:35:11.631 INFO:tasks.workunit.client.0.smithi026.stdout:201 2024-09-13T14:35:11.632 INFO:tasks.workunit.client.0.smithi026.stdout:202 2024-09-13T14:35:11.633 INFO:tasks.workunit.client.0.smithi026.stdout:203 2024-09-13T14:35:11.634 INFO:tasks.workunit.client.0.smithi026.stdout:204 2024-09-13T14:35:11.635 INFO:tasks.workunit.client.0.smithi026.stdout:205 2024-09-13T14:35:11.636 INFO:tasks.workunit.client.0.smithi026.stdout:206 2024-09-13T14:35:11.637 INFO:tasks.workunit.client.0.smithi026.stdout:207 2024-09-13T14:35:11.638 INFO:tasks.workunit.client.0.smithi026.stdout:208 2024-09-13T14:35:11.639 INFO:tasks.workunit.client.0.smithi026.stdout:209 2024-09-13T14:35:11.640 INFO:tasks.workunit.client.0.smithi026.stdout:210 2024-09-13T14:35:11.641 INFO:tasks.workunit.client.0.smithi026.stdout:211 2024-09-13T14:35:11.642 INFO:tasks.workunit.client.0.smithi026.stdout:212 2024-09-13T14:35:11.643 INFO:tasks.workunit.client.0.smithi026.stdout:213 2024-09-13T14:35:11.644 INFO:tasks.workunit.client.0.smithi026.stdout:214 2024-09-13T14:35:11.645 INFO:tasks.workunit.client.0.smithi026.stdout:215 2024-09-13T14:35:11.646 INFO:tasks.workunit.client.0.smithi026.stdout:216 2024-09-13T14:35:11.647 INFO:tasks.workunit.client.0.smithi026.stdout:217 2024-09-13T14:35:11.648 INFO:tasks.workunit.client.0.smithi026.stdout:218 2024-09-13T14:35:11.649 INFO:tasks.workunit.client.0.smithi026.stdout:219 2024-09-13T14:35:11.650 INFO:tasks.workunit.client.0.smithi026.stdout:220 2024-09-13T14:35:11.651 INFO:tasks.workunit.client.0.smithi026.stdout:221 2024-09-13T14:35:11.652 INFO:tasks.workunit.client.0.smithi026.stdout:222 2024-09-13T14:35:11.653 INFO:tasks.workunit.client.0.smithi026.stdout:223 2024-09-13T14:35:11.654 INFO:tasks.workunit.client.0.smithi026.stdout:224 2024-09-13T14:35:11.655 INFO:tasks.workunit.client.0.smithi026.stdout:225 2024-09-13T14:35:11.656 INFO:tasks.workunit.client.0.smithi026.stdout:226 2024-09-13T14:35:11.657 INFO:tasks.workunit.client.0.smithi026.stdout:227 2024-09-13T14:35:11.658 INFO:tasks.workunit.client.0.smithi026.stdout:228 2024-09-13T14:35:11.659 INFO:tasks.workunit.client.0.smithi026.stdout:229 2024-09-13T14:35:11.661 INFO:tasks.workunit.client.0.smithi026.stdout:230 2024-09-13T14:35:11.662 INFO:tasks.workunit.client.0.smithi026.stdout:231 2024-09-13T14:35:11.663 INFO:tasks.workunit.client.0.smithi026.stdout:232 2024-09-13T14:35:11.664 INFO:tasks.workunit.client.0.smithi026.stdout:233 2024-09-13T14:35:11.665 INFO:tasks.workunit.client.0.smithi026.stdout:234 2024-09-13T14:35:11.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:11] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-13T14:35:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[97022]: osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[99461]: osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:11.666 INFO:tasks.workunit.client.0.smithi026.stdout:235 2024-09-13T14:35:11.667 INFO:tasks.workunit.client.0.smithi026.stdout:236 2024-09-13T14:35:11.668 INFO:tasks.workunit.client.0.smithi026.stdout:237 2024-09-13T14:35:11.669 INFO:tasks.workunit.client.0.smithi026.stdout:238 2024-09-13T14:35:11.670 INFO:tasks.workunit.client.0.smithi026.stdout:239 2024-09-13T14:35:11.671 INFO:tasks.workunit.client.0.smithi026.stdout:240 2024-09-13T14:35:11.672 INFO:tasks.workunit.client.0.smithi026.stdout:241 2024-09-13T14:35:11.673 INFO:tasks.workunit.client.0.smithi026.stdout:242 2024-09-13T14:35:11.674 INFO:tasks.workunit.client.0.smithi026.stdout:243 2024-09-13T14:35:11.675 INFO:tasks.workunit.client.0.smithi026.stdout:244 2024-09-13T14:35:11.676 INFO:tasks.workunit.client.0.smithi026.stdout:245 2024-09-13T14:35:11.677 INFO:tasks.workunit.client.0.smithi026.stdout:246 2024-09-13T14:35:11.678 INFO:tasks.workunit.client.0.smithi026.stdout:247 2024-09-13T14:35:11.680 INFO:tasks.workunit.client.0.smithi026.stdout:248 2024-09-13T14:35:11.681 INFO:tasks.workunit.client.0.smithi026.stdout:249 2024-09-13T14:35:11.682 INFO:tasks.workunit.client.0.smithi026.stdout:250 2024-09-13T14:35:11.683 INFO:tasks.workunit.client.0.smithi026.stdout:251 2024-09-13T14:35:11.684 INFO:tasks.workunit.client.0.smithi026.stdout:252 2024-09-13T14:35:11.685 INFO:tasks.workunit.client.0.smithi026.stdout:253 2024-09-13T14:35:11.686 INFO:tasks.workunit.client.0.smithi026.stdout:254 2024-09-13T14:35:11.687 INFO:tasks.workunit.client.0.smithi026.stdout:255 2024-09-13T14:35:11.689 INFO:tasks.workunit.client.0.smithi026.stdout:256 2024-09-13T14:35:11.690 INFO:tasks.workunit.client.0.smithi026.stdout:257 2024-09-13T14:35:11.691 INFO:tasks.workunit.client.0.smithi026.stdout:258 2024-09-13T14:35:11.692 INFO:tasks.workunit.client.0.smithi026.stdout:259 2024-09-13T14:35:11.693 INFO:tasks.workunit.client.0.smithi026.stdout:260 2024-09-13T14:35:11.694 INFO:tasks.workunit.client.0.smithi026.stdout:261 2024-09-13T14:35:11.695 INFO:tasks.workunit.client.0.smithi026.stdout:262 2024-09-13T14:35:11.696 INFO:tasks.workunit.client.0.smithi026.stdout:263 2024-09-13T14:35:11.697 INFO:tasks.workunit.client.0.smithi026.stdout:264 2024-09-13T14:35:11.698 INFO:tasks.workunit.client.0.smithi026.stdout:265 2024-09-13T14:35:11.700 INFO:tasks.workunit.client.0.smithi026.stdout:266 2024-09-13T14:35:11.701 INFO:tasks.workunit.client.0.smithi026.stdout:267 2024-09-13T14:35:11.702 INFO:tasks.workunit.client.0.smithi026.stdout:268 2024-09-13T14:35:11.703 INFO:tasks.workunit.client.0.smithi026.stdout:269 2024-09-13T14:35:11.704 INFO:tasks.workunit.client.0.smithi026.stdout:270 2024-09-13T14:35:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:11 smithi145 ceph-mon[89145]: osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:35:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:11 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:11 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:11.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:11.705 INFO:tasks.workunit.client.0.smithi026.stdout:271 2024-09-13T14:35:11.706 INFO:tasks.workunit.client.0.smithi026.stdout:272 2024-09-13T14:35:11.707 INFO:tasks.workunit.client.0.smithi026.stdout:273 2024-09-13T14:35:11.708 INFO:tasks.workunit.client.0.smithi026.stdout:274 2024-09-13T14:35:11.709 INFO:tasks.workunit.client.0.smithi026.stdout:275 2024-09-13T14:35:11.710 INFO:tasks.workunit.client.0.smithi026.stdout:276 2024-09-13T14:35:11.712 INFO:tasks.workunit.client.0.smithi026.stdout:277 2024-09-13T14:35:11.713 INFO:tasks.workunit.client.0.smithi026.stdout:278 2024-09-13T14:35:11.714 INFO:tasks.workunit.client.0.smithi026.stdout:279 2024-09-13T14:35:11.715 INFO:tasks.workunit.client.0.smithi026.stdout:280 2024-09-13T14:35:11.716 INFO:tasks.workunit.client.0.smithi026.stdout:281 2024-09-13T14:35:11.717 INFO:tasks.workunit.client.0.smithi026.stdout:282 2024-09-13T14:35:11.718 INFO:tasks.workunit.client.0.smithi026.stdout:283 2024-09-13T14:35:11.719 INFO:tasks.workunit.client.0.smithi026.stdout:284 2024-09-13T14:35:11.720 INFO:tasks.workunit.client.0.smithi026.stdout:285 2024-09-13T14:35:11.721 INFO:tasks.workunit.client.0.smithi026.stdout:286 2024-09-13T14:35:11.722 INFO:tasks.workunit.client.0.smithi026.stdout:287 2024-09-13T14:35:11.723 INFO:tasks.workunit.client.0.smithi026.stdout:288 2024-09-13T14:35:11.725 INFO:tasks.workunit.client.0.smithi026.stdout:289 2024-09-13T14:35:11.726 INFO:tasks.workunit.client.0.smithi026.stdout:290 2024-09-13T14:35:11.727 INFO:tasks.workunit.client.0.smithi026.stdout:291 2024-09-13T14:35:11.728 INFO:tasks.workunit.client.0.smithi026.stdout:292 2024-09-13T14:35:11.729 INFO:tasks.workunit.client.0.smithi026.stdout:293 2024-09-13T14:35:11.730 INFO:tasks.workunit.client.0.smithi026.stdout:294 2024-09-13T14:35:11.731 INFO:tasks.workunit.client.0.smithi026.stdout:295 2024-09-13T14:35:11.732 INFO:tasks.workunit.client.0.smithi026.stdout:296 2024-09-13T14:35:11.733 INFO:tasks.workunit.client.0.smithi026.stdout:297 2024-09-13T14:35:11.734 INFO:tasks.workunit.client.0.smithi026.stdout:298 2024-09-13T14:35:11.735 INFO:tasks.workunit.client.0.smithi026.stdout:299 2024-09-13T14:35:11.736 INFO:tasks.workunit.client.0.smithi026.stdout:300 2024-09-13T14:35:11.738 INFO:tasks.workunit.client.0.smithi026.stdout:301 2024-09-13T14:35:11.739 INFO:tasks.workunit.client.0.smithi026.stdout:302 2024-09-13T14:35:11.740 INFO:tasks.workunit.client.0.smithi026.stdout:303 2024-09-13T14:35:11.741 INFO:tasks.workunit.client.0.smithi026.stdout:304 2024-09-13T14:35:11.742 INFO:tasks.workunit.client.0.smithi026.stdout:305 2024-09-13T14:35:11.743 INFO:tasks.workunit.client.0.smithi026.stdout:306 2024-09-13T14:35:11.744 INFO:tasks.workunit.client.0.smithi026.stdout:307 2024-09-13T14:35:11.745 INFO:tasks.workunit.client.0.smithi026.stdout:308 2024-09-13T14:35:11.746 INFO:tasks.workunit.client.0.smithi026.stdout:309 2024-09-13T14:35:11.747 INFO:tasks.workunit.client.0.smithi026.stdout:310 2024-09-13T14:35:11.748 INFO:tasks.workunit.client.0.smithi026.stdout:311 2024-09-13T14:35:11.749 INFO:tasks.workunit.client.0.smithi026.stdout:312 2024-09-13T14:35:11.751 INFO:tasks.workunit.client.0.smithi026.stdout:313 2024-09-13T14:35:11.752 INFO:tasks.workunit.client.0.smithi026.stdout:314 2024-09-13T14:35:11.753 INFO:tasks.workunit.client.0.smithi026.stdout:315 2024-09-13T14:35:11.754 INFO:tasks.workunit.client.0.smithi026.stdout:316 2024-09-13T14:35:11.755 INFO:tasks.workunit.client.0.smithi026.stdout:317 2024-09-13T14:35:11.756 INFO:tasks.workunit.client.0.smithi026.stdout:318 2024-09-13T14:35:11.757 INFO:tasks.workunit.client.0.smithi026.stdout:319 2024-09-13T14:35:11.758 INFO:tasks.workunit.client.0.smithi026.stdout:320 2024-09-13T14:35:11.759 INFO:tasks.workunit.client.0.smithi026.stdout:321 2024-09-13T14:35:11.760 INFO:tasks.workunit.client.0.smithi026.stdout:322 2024-09-13T14:35:11.761 INFO:tasks.workunit.client.0.smithi026.stdout:323 2024-09-13T14:35:11.762 INFO:tasks.workunit.client.0.smithi026.stdout:324 2024-09-13T14:35:11.763 INFO:tasks.workunit.client.0.smithi026.stdout:325 2024-09-13T14:35:11.764 INFO:tasks.workunit.client.0.smithi026.stdout:326 2024-09-13T14:35:11.765 INFO:tasks.workunit.client.0.smithi026.stdout:327 2024-09-13T14:35:11.766 INFO:tasks.workunit.client.0.smithi026.stdout:328 2024-09-13T14:35:11.768 INFO:tasks.workunit.client.0.smithi026.stdout:329 2024-09-13T14:35:11.769 INFO:tasks.workunit.client.0.smithi026.stdout:330 2024-09-13T14:35:11.770 INFO:tasks.workunit.client.0.smithi026.stdout:331 2024-09-13T14:35:11.770 INFO:tasks.workunit.client.0.smithi026.stdout:332 2024-09-13T14:35:11.771 INFO:tasks.workunit.client.0.smithi026.stdout:333 2024-09-13T14:35:11.772 INFO:tasks.workunit.client.0.smithi026.stdout:334 2024-09-13T14:35:11.773 INFO:tasks.workunit.client.0.smithi026.stdout:335 2024-09-13T14:35:11.774 INFO:tasks.workunit.client.0.smithi026.stdout:336 2024-09-13T14:35:11.775 INFO:tasks.workunit.client.0.smithi026.stdout:337 2024-09-13T14:35:11.776 INFO:tasks.workunit.client.0.smithi026.stdout:338 2024-09-13T14:35:11.777 INFO:tasks.workunit.client.0.smithi026.stdout:339 2024-09-13T14:35:11.778 INFO:tasks.workunit.client.0.smithi026.stdout:340 2024-09-13T14:35:11.779 INFO:tasks.workunit.client.0.smithi026.stdout:341 2024-09-13T14:35:11.780 INFO:tasks.workunit.client.0.smithi026.stdout:342 2024-09-13T14:35:11.781 INFO:tasks.workunit.client.0.smithi026.stdout:343 2024-09-13T14:35:11.782 INFO:tasks.workunit.client.0.smithi026.stdout:344 2024-09-13T14:35:11.783 INFO:tasks.workunit.client.0.smithi026.stdout:345 2024-09-13T14:35:11.784 INFO:tasks.workunit.client.0.smithi026.stdout:346 2024-09-13T14:35:11.785 INFO:tasks.workunit.client.0.smithi026.stdout:347 2024-09-13T14:35:11.785 INFO:tasks.workunit.client.0.smithi026.stdout:348 2024-09-13T14:35:11.786 INFO:tasks.workunit.client.0.smithi026.stdout:349 2024-09-13T14:35:11.787 INFO:tasks.workunit.client.0.smithi026.stdout:350 2024-09-13T14:35:11.788 INFO:tasks.workunit.client.0.smithi026.stdout:351 2024-09-13T14:35:11.789 INFO:tasks.workunit.client.0.smithi026.stdout:352 2024-09-13T14:35:11.790 INFO:tasks.workunit.client.0.smithi026.stdout:353 2024-09-13T14:35:11.791 INFO:tasks.workunit.client.0.smithi026.stdout:354 2024-09-13T14:35:11.792 INFO:tasks.workunit.client.0.smithi026.stdout:355 2024-09-13T14:35:11.793 INFO:tasks.workunit.client.0.smithi026.stdout:356 2024-09-13T14:35:11.794 INFO:tasks.workunit.client.0.smithi026.stdout:357 2024-09-13T14:35:11.795 INFO:tasks.workunit.client.0.smithi026.stdout:358 2024-09-13T14:35:11.796 INFO:tasks.workunit.client.0.smithi026.stdout:359 2024-09-13T14:35:11.797 INFO:tasks.workunit.client.0.smithi026.stdout:360 2024-09-13T14:35:11.798 INFO:tasks.workunit.client.0.smithi026.stdout:361 2024-09-13T14:35:11.799 INFO:tasks.workunit.client.0.smithi026.stdout:362 2024-09-13T14:35:11.800 INFO:tasks.workunit.client.0.smithi026.stdout:363 2024-09-13T14:35:11.801 INFO:tasks.workunit.client.0.smithi026.stdout:364 2024-09-13T14:35:11.802 INFO:tasks.workunit.client.0.smithi026.stdout:365 2024-09-13T14:35:11.803 INFO:tasks.workunit.client.0.smithi026.stdout:366 2024-09-13T14:35:11.804 INFO:tasks.workunit.client.0.smithi026.stdout:367 2024-09-13T14:35:11.805 INFO:tasks.workunit.client.0.smithi026.stdout:368 2024-09-13T14:35:11.806 INFO:tasks.workunit.client.0.smithi026.stdout:369 2024-09-13T14:35:11.807 INFO:tasks.workunit.client.0.smithi026.stdout:370 2024-09-13T14:35:11.807 INFO:tasks.workunit.client.0.smithi026.stdout:371 2024-09-13T14:35:11.809 INFO:tasks.workunit.client.0.smithi026.stdout:372 2024-09-13T14:35:11.809 INFO:tasks.workunit.client.0.smithi026.stdout:373 2024-09-13T14:35:11.811 INFO:tasks.workunit.client.0.smithi026.stdout:374 2024-09-13T14:35:11.812 INFO:tasks.workunit.client.0.smithi026.stdout:375 2024-09-13T14:35:11.813 INFO:tasks.workunit.client.0.smithi026.stdout:376 2024-09-13T14:35:11.813 INFO:tasks.workunit.client.0.smithi026.stdout:377 2024-09-13T14:35:11.814 INFO:tasks.workunit.client.0.smithi026.stdout:378 2024-09-13T14:35:11.815 INFO:tasks.workunit.client.0.smithi026.stdout:379 2024-09-13T14:35:11.817 INFO:tasks.workunit.client.0.smithi026.stdout:380 2024-09-13T14:35:11.818 INFO:tasks.workunit.client.0.smithi026.stdout:381 2024-09-13T14:35:11.818 INFO:tasks.workunit.client.0.smithi026.stdout:382 2024-09-13T14:35:11.819 INFO:tasks.workunit.client.0.smithi026.stdout:383 2024-09-13T14:35:11.821 INFO:tasks.workunit.client.0.smithi026.stdout:384 2024-09-13T14:35:11.822 INFO:tasks.workunit.client.0.smithi026.stdout:385 2024-09-13T14:35:11.823 INFO:tasks.workunit.client.0.smithi026.stdout:386 2024-09-13T14:35:11.823 INFO:tasks.workunit.client.0.smithi026.stdout:387 2024-09-13T14:35:11.824 INFO:tasks.workunit.client.0.smithi026.stdout:388 2024-09-13T14:35:11.825 INFO:tasks.workunit.client.0.smithi026.stdout:389 2024-09-13T14:35:11.826 INFO:tasks.workunit.client.0.smithi026.stdout:390 2024-09-13T14:35:11.827 INFO:tasks.workunit.client.0.smithi026.stdout:391 2024-09-13T14:35:11.828 INFO:tasks.workunit.client.0.smithi026.stdout:392 2024-09-13T14:35:11.829 INFO:tasks.workunit.client.0.smithi026.stdout:393 2024-09-13T14:35:11.830 INFO:tasks.workunit.client.0.smithi026.stdout:394 2024-09-13T14:35:11.831 INFO:tasks.workunit.client.0.smithi026.stdout:395 2024-09-13T14:35:11.832 INFO:tasks.workunit.client.0.smithi026.stdout:396 2024-09-13T14:35:11.833 INFO:tasks.workunit.client.0.smithi026.stdout:397 2024-09-13T14:35:11.834 INFO:tasks.workunit.client.0.smithi026.stdout:398 2024-09-13T14:35:11.835 INFO:tasks.workunit.client.0.smithi026.stdout:399 2024-09-13T14:35:11.836 INFO:tasks.workunit.client.0.smithi026.stdout:400 2024-09-13T14:35:11.837 INFO:tasks.workunit.client.0.smithi026.stdout:401 2024-09-13T14:35:11.838 INFO:tasks.workunit.client.0.smithi026.stdout:402 2024-09-13T14:35:11.838 INFO:tasks.workunit.client.0.smithi026.stdout:403 2024-09-13T14:35:11.839 INFO:tasks.workunit.client.0.smithi026.stdout:404 2024-09-13T14:35:11.840 INFO:tasks.workunit.client.0.smithi026.stdout:405 2024-09-13T14:35:11.841 INFO:tasks.workunit.client.0.smithi026.stdout:406 2024-09-13T14:35:11.842 INFO:tasks.workunit.client.0.smithi026.stdout:407 2024-09-13T14:35:11.843 INFO:tasks.workunit.client.0.smithi026.stdout:408 2024-09-13T14:35:11.844 INFO:tasks.workunit.client.0.smithi026.stdout:409 2024-09-13T14:35:11.845 INFO:tasks.workunit.client.0.smithi026.stdout:410 2024-09-13T14:35:11.846 INFO:tasks.workunit.client.0.smithi026.stdout:411 2024-09-13T14:35:11.847 INFO:tasks.workunit.client.0.smithi026.stdout:412 2024-09-13T14:35:11.848 INFO:tasks.workunit.client.0.smithi026.stdout:413 2024-09-13T14:35:11.849 INFO:tasks.workunit.client.0.smithi026.stdout:414 2024-09-13T14:35:11.851 INFO:tasks.workunit.client.0.smithi026.stdout:415 2024-09-13T14:35:11.852 INFO:tasks.workunit.client.0.smithi026.stdout:416 2024-09-13T14:35:11.853 INFO:tasks.workunit.client.0.smithi026.stdout:417 2024-09-13T14:35:11.854 INFO:tasks.workunit.client.0.smithi026.stdout:418 2024-09-13T14:35:11.855 INFO:tasks.workunit.client.0.smithi026.stdout:419 2024-09-13T14:35:11.855 INFO:tasks.workunit.client.0.smithi026.stdout:420 2024-09-13T14:35:11.856 INFO:tasks.workunit.client.0.smithi026.stdout:421 2024-09-13T14:35:11.857 INFO:tasks.workunit.client.0.smithi026.stdout:422 2024-09-13T14:35:11.858 INFO:tasks.workunit.client.0.smithi026.stdout:423 2024-09-13T14:35:11.862 INFO:tasks.workunit.client.0.smithi026.stdout:424 2024-09-13T14:35:11.863 INFO:tasks.workunit.client.0.smithi026.stdout:425 2024-09-13T14:35:11.864 INFO:tasks.workunit.client.0.smithi026.stdout:426 2024-09-13T14:35:11.865 INFO:tasks.workunit.client.0.smithi026.stdout:427 2024-09-13T14:35:11.866 INFO:tasks.workunit.client.0.smithi026.stdout:428 2024-09-13T14:35:11.867 INFO:tasks.workunit.client.0.smithi026.stdout:429 2024-09-13T14:35:11.868 INFO:tasks.workunit.client.0.smithi026.stdout:430 2024-09-13T14:35:11.869 INFO:tasks.workunit.client.0.smithi026.stdout:431 2024-09-13T14:35:11.870 INFO:tasks.workunit.client.0.smithi026.stdout:432 2024-09-13T14:35:11.870 INFO:tasks.workunit.client.0.smithi026.stdout:433 2024-09-13T14:35:11.871 INFO:tasks.workunit.client.0.smithi026.stdout:434 2024-09-13T14:35:11.872 INFO:tasks.workunit.client.0.smithi026.stdout:435 2024-09-13T14:35:11.873 INFO:tasks.workunit.client.0.smithi026.stdout:436 2024-09-13T14:35:11.874 INFO:tasks.workunit.client.0.smithi026.stdout:437 2024-09-13T14:35:11.875 INFO:tasks.workunit.client.0.smithi026.stdout:438 2024-09-13T14:35:11.876 INFO:tasks.workunit.client.0.smithi026.stdout:439 2024-09-13T14:35:11.877 INFO:tasks.workunit.client.0.smithi026.stdout:440 2024-09-13T14:35:11.878 INFO:tasks.workunit.client.0.smithi026.stdout:441 2024-09-13T14:35:11.879 INFO:tasks.workunit.client.0.smithi026.stdout:442 2024-09-13T14:35:11.880 INFO:tasks.workunit.client.0.smithi026.stdout:443 2024-09-13T14:35:11.881 INFO:tasks.workunit.client.0.smithi026.stdout:444 2024-09-13T14:35:11.882 INFO:tasks.workunit.client.0.smithi026.stdout:445 2024-09-13T14:35:11.882 INFO:tasks.workunit.client.0.smithi026.stdout:446 2024-09-13T14:35:11.883 INFO:tasks.workunit.client.0.smithi026.stdout:447 2024-09-13T14:35:11.884 INFO:tasks.workunit.client.0.smithi026.stdout:448 2024-09-13T14:35:11.885 INFO:tasks.workunit.client.0.smithi026.stdout:449 2024-09-13T14:35:11.886 INFO:tasks.workunit.client.0.smithi026.stdout:450 2024-09-13T14:35:11.887 INFO:tasks.workunit.client.0.smithi026.stdout:451 2024-09-13T14:35:11.888 INFO:tasks.workunit.client.0.smithi026.stdout:452 2024-09-13T14:35:11.889 INFO:tasks.workunit.client.0.smithi026.stdout:453 2024-09-13T14:35:11.890 INFO:tasks.workunit.client.0.smithi026.stdout:454 2024-09-13T14:35:11.891 INFO:tasks.workunit.client.0.smithi026.stdout:455 2024-09-13T14:35:11.892 INFO:tasks.workunit.client.0.smithi026.stdout:456 2024-09-13T14:35:11.892 INFO:tasks.workunit.client.0.smithi026.stdout:457 2024-09-13T14:35:11.893 INFO:tasks.workunit.client.0.smithi026.stdout:458 2024-09-13T14:35:11.894 INFO:tasks.workunit.client.0.smithi026.stdout:459 2024-09-13T14:35:11.895 INFO:tasks.workunit.client.0.smithi026.stdout:460 2024-09-13T14:35:11.897 INFO:tasks.workunit.client.0.smithi026.stdout:461 2024-09-13T14:35:11.898 INFO:tasks.workunit.client.0.smithi026.stdout:462 2024-09-13T14:35:11.899 INFO:tasks.workunit.client.0.smithi026.stdout:463 2024-09-13T14:35:11.900 INFO:tasks.workunit.client.0.smithi026.stdout:464 2024-09-13T14:35:11.901 INFO:tasks.workunit.client.0.smithi026.stdout:465 2024-09-13T14:35:11.902 INFO:tasks.workunit.client.0.smithi026.stdout:466 2024-09-13T14:35:11.903 INFO:tasks.workunit.client.0.smithi026.stdout:467 2024-09-13T14:35:11.904 INFO:tasks.workunit.client.0.smithi026.stdout:468 2024-09-13T14:35:11.905 INFO:tasks.workunit.client.0.smithi026.stdout:469 2024-09-13T14:35:11.906 INFO:tasks.workunit.client.0.smithi026.stdout:470 2024-09-13T14:35:11.907 INFO:tasks.workunit.client.0.smithi026.stdout:471 2024-09-13T14:35:11.908 INFO:tasks.workunit.client.0.smithi026.stdout:472 2024-09-13T14:35:11.909 INFO:tasks.workunit.client.0.smithi026.stdout:473 2024-09-13T14:35:11.910 INFO:tasks.workunit.client.0.smithi026.stdout:474 2024-09-13T14:35:11.911 INFO:tasks.workunit.client.0.smithi026.stdout:475 2024-09-13T14:35:11.912 INFO:tasks.workunit.client.0.smithi026.stdout:476 2024-09-13T14:35:11.913 INFO:tasks.workunit.client.0.smithi026.stdout:477 2024-09-13T14:35:11.914 INFO:tasks.workunit.client.0.smithi026.stdout:478 2024-09-13T14:35:11.915 INFO:tasks.workunit.client.0.smithi026.stdout:479 2024-09-13T14:35:11.916 INFO:tasks.workunit.client.0.smithi026.stdout:480 2024-09-13T14:35:11.917 INFO:tasks.workunit.client.0.smithi026.stdout:481 2024-09-13T14:35:11.917 INFO:tasks.workunit.client.0.smithi026.stdout:482 2024-09-13T14:35:11.918 INFO:tasks.workunit.client.0.smithi026.stdout:483 2024-09-13T14:35:11.919 INFO:tasks.workunit.client.0.smithi026.stdout:484 2024-09-13T14:35:11.920 INFO:tasks.workunit.client.0.smithi026.stdout:485 2024-09-13T14:35:11.921 INFO:tasks.workunit.client.0.smithi026.stdout:486 2024-09-13T14:35:11.922 INFO:tasks.workunit.client.0.smithi026.stdout:487 2024-09-13T14:35:11.924 INFO:tasks.workunit.client.0.smithi026.stdout:488 2024-09-13T14:35:11.924 INFO:tasks.workunit.client.0.smithi026.stdout:489 2024-09-13T14:35:11.926 INFO:tasks.workunit.client.0.smithi026.stdout:490 2024-09-13T14:35:11.926 INFO:tasks.workunit.client.0.smithi026.stdout:491 2024-09-13T14:35:11.927 INFO:tasks.workunit.client.0.smithi026.stdout:492 2024-09-13T14:35:11.928 INFO:tasks.workunit.client.0.smithi026.stdout:493 2024-09-13T14:35:11.929 INFO:tasks.workunit.client.0.smithi026.stdout:494 2024-09-13T14:35:11.930 INFO:tasks.workunit.client.0.smithi026.stdout:495 2024-09-13T14:35:11.931 INFO:tasks.workunit.client.0.smithi026.stdout:496 2024-09-13T14:35:11.932 INFO:tasks.workunit.client.0.smithi026.stdout:497 2024-09-13T14:35:11.933 INFO:tasks.workunit.client.0.smithi026.stdout:498 2024-09-13T14:35:11.934 INFO:tasks.workunit.client.0.smithi026.stdout:499 2024-09-13T14:35:11.935 INFO:tasks.workunit.client.0.smithi026.stdout:500 2024-09-13T14:35:11.936 INFO:tasks.workunit.client.0.smithi026.stdout:501 2024-09-13T14:35:11.937 INFO:tasks.workunit.client.0.smithi026.stdout:502 2024-09-13T14:35:11.938 INFO:tasks.workunit.client.0.smithi026.stdout:503 2024-09-13T14:35:11.939 INFO:tasks.workunit.client.0.smithi026.stdout:504 2024-09-13T14:35:11.940 INFO:tasks.workunit.client.0.smithi026.stdout:505 2024-09-13T14:35:11.941 INFO:tasks.workunit.client.0.smithi026.stdout:506 2024-09-13T14:35:11.942 INFO:tasks.workunit.client.0.smithi026.stdout:507 2024-09-13T14:35:11.943 INFO:tasks.workunit.client.0.smithi026.stdout:508 2024-09-13T14:35:11.944 INFO:tasks.workunit.client.0.smithi026.stdout:509 2024-09-13T14:35:11.945 INFO:tasks.workunit.client.0.smithi026.stdout:510 2024-09-13T14:35:11.946 INFO:tasks.workunit.client.0.smithi026.stdout:511 2024-09-13T14:35:11.947 INFO:tasks.workunit.client.0.smithi026.stdout:512 2024-09-13T14:35:11.948 INFO:tasks.workunit.client.0.smithi026.stdout:513 2024-09-13T14:35:11.949 INFO:tasks.workunit.client.0.smithi026.stdout:514 2024-09-13T14:35:11.950 INFO:tasks.workunit.client.0.smithi026.stdout:515 2024-09-13T14:35:11.951 INFO:tasks.workunit.client.0.smithi026.stdout:516 2024-09-13T14:35:11.952 INFO:tasks.workunit.client.0.smithi026.stdout:517 2024-09-13T14:35:11.954 INFO:tasks.workunit.client.0.smithi026.stdout:518 2024-09-13T14:35:11.955 INFO:tasks.workunit.client.0.smithi026.stdout:519 2024-09-13T14:35:11.956 INFO:tasks.workunit.client.0.smithi026.stdout:520 2024-09-13T14:35:11.957 INFO:tasks.workunit.client.0.smithi026.stdout:521 2024-09-13T14:35:11.958 INFO:tasks.workunit.client.0.smithi026.stdout:522 2024-09-13T14:35:11.959 INFO:tasks.workunit.client.0.smithi026.stdout:523 2024-09-13T14:35:11.960 INFO:tasks.workunit.client.0.smithi026.stdout:524 2024-09-13T14:35:11.961 INFO:tasks.workunit.client.0.smithi026.stdout:525 2024-09-13T14:35:11.962 INFO:tasks.workunit.client.0.smithi026.stdout:526 2024-09-13T14:35:11.963 INFO:tasks.workunit.client.0.smithi026.stdout:527 2024-09-13T14:35:11.964 INFO:tasks.workunit.client.0.smithi026.stdout:528 2024-09-13T14:35:11.965 INFO:tasks.workunit.client.0.smithi026.stdout:529 2024-09-13T14:35:11.968 INFO:tasks.workunit.client.0.smithi026.stdout:530 2024-09-13T14:35:11.968 INFO:tasks.workunit.client.0.smithi026.stdout:531 2024-09-13T14:35:11.968 INFO:tasks.workunit.client.0.smithi026.stdout:532 2024-09-13T14:35:11.969 INFO:tasks.workunit.client.0.smithi026.stdout:533 2024-09-13T14:35:11.969 INFO:tasks.workunit.client.0.smithi026.stdout:534 2024-09-13T14:35:11.970 INFO:tasks.workunit.client.0.smithi026.stdout:535 2024-09-13T14:35:11.972 INFO:tasks.workunit.client.0.smithi026.stdout:536 2024-09-13T14:35:11.972 INFO:tasks.workunit.client.0.smithi026.stdout:537 2024-09-13T14:35:11.973 INFO:tasks.workunit.client.0.smithi026.stdout:538 2024-09-13T14:35:11.974 INFO:tasks.workunit.client.0.smithi026.stdout:539 2024-09-13T14:35:11.975 INFO:tasks.workunit.client.0.smithi026.stdout:540 2024-09-13T14:35:11.976 INFO:tasks.workunit.client.0.smithi026.stdout:541 2024-09-13T14:35:11.977 INFO:tasks.workunit.client.0.smithi026.stdout:542 2024-09-13T14:35:11.978 INFO:tasks.workunit.client.0.smithi026.stdout:543 2024-09-13T14:35:11.979 INFO:tasks.workunit.client.0.smithi026.stdout:544 2024-09-13T14:35:11.980 INFO:tasks.workunit.client.0.smithi026.stdout:545 2024-09-13T14:35:11.982 INFO:tasks.workunit.client.0.smithi026.stdout:546 2024-09-13T14:35:11.982 INFO:tasks.workunit.client.0.smithi026.stdout:547 2024-09-13T14:35:11.983 INFO:tasks.workunit.client.0.smithi026.stdout:548 2024-09-13T14:35:11.984 INFO:tasks.workunit.client.0.smithi026.stdout:549 2024-09-13T14:35:11.985 INFO:tasks.workunit.client.0.smithi026.stdout:550 2024-09-13T14:35:11.986 INFO:tasks.workunit.client.0.smithi026.stdout:551 2024-09-13T14:35:11.987 INFO:tasks.workunit.client.0.smithi026.stdout:552 2024-09-13T14:35:11.988 INFO:tasks.workunit.client.0.smithi026.stdout:553 2024-09-13T14:35:11.989 INFO:tasks.workunit.client.0.smithi026.stdout:554 2024-09-13T14:35:11.990 INFO:tasks.workunit.client.0.smithi026.stdout:555 2024-09-13T14:35:11.991 INFO:tasks.workunit.client.0.smithi026.stdout:556 2024-09-13T14:35:11.992 INFO:tasks.workunit.client.0.smithi026.stdout:557 2024-09-13T14:35:11.993 INFO:tasks.workunit.client.0.smithi026.stdout:558 2024-09-13T14:35:11.994 INFO:tasks.workunit.client.0.smithi026.stdout:559 2024-09-13T14:35:11.995 INFO:tasks.workunit.client.0.smithi026.stdout:560 2024-09-13T14:35:11.996 INFO:tasks.workunit.client.0.smithi026.stdout:561 2024-09-13T14:35:11.997 INFO:tasks.workunit.client.0.smithi026.stdout:562 2024-09-13T14:35:11.998 INFO:tasks.workunit.client.0.smithi026.stdout:563 2024-09-13T14:35:11.999 INFO:tasks.workunit.client.0.smithi026.stdout:564 2024-09-13T14:35:12.000 INFO:tasks.workunit.client.0.smithi026.stdout:565 2024-09-13T14:35:12.001 INFO:tasks.workunit.client.0.smithi026.stdout:566 2024-09-13T14:35:12.002 INFO:tasks.workunit.client.0.smithi026.stdout:567 2024-09-13T14:35:12.003 INFO:tasks.workunit.client.0.smithi026.stdout:568 2024-09-13T14:35:12.004 INFO:tasks.workunit.client.0.smithi026.stdout:569 2024-09-13T14:35:12.005 INFO:tasks.workunit.client.0.smithi026.stdout:570 2024-09-13T14:35:12.006 INFO:tasks.workunit.client.0.smithi026.stdout:571 2024-09-13T14:35:12.007 INFO:tasks.workunit.client.0.smithi026.stdout:572 2024-09-13T14:35:12.008 INFO:tasks.workunit.client.0.smithi026.stdout:573 2024-09-13T14:35:12.009 INFO:tasks.workunit.client.0.smithi026.stdout:574 2024-09-13T14:35:12.010 INFO:tasks.workunit.client.0.smithi026.stdout:575 2024-09-13T14:35:12.011 INFO:tasks.workunit.client.0.smithi026.stdout:576 2024-09-13T14:35:12.012 INFO:tasks.workunit.client.0.smithi026.stdout:577 2024-09-13T14:35:12.013 INFO:tasks.workunit.client.0.smithi026.stdout:578 2024-09-13T14:35:12.014 INFO:tasks.workunit.client.0.smithi026.stdout:579 2024-09-13T14:35:12.015 INFO:tasks.workunit.client.0.smithi026.stdout:580 2024-09-13T14:35:12.016 INFO:tasks.workunit.client.0.smithi026.stdout:581 2024-09-13T14:35:12.017 INFO:tasks.workunit.client.0.smithi026.stdout:582 2024-09-13T14:35:12.019 INFO:tasks.workunit.client.0.smithi026.stdout:583 2024-09-13T14:35:12.020 INFO:tasks.workunit.client.0.smithi026.stdout:584 2024-09-13T14:35:12.021 INFO:tasks.workunit.client.0.smithi026.stdout:585 2024-09-13T14:35:12.022 INFO:tasks.workunit.client.0.smithi026.stdout:586 2024-09-13T14:35:12.023 INFO:tasks.workunit.client.0.smithi026.stdout:587 2024-09-13T14:35:12.024 INFO:tasks.workunit.client.0.smithi026.stdout:588 2024-09-13T14:35:12.025 INFO:tasks.workunit.client.0.smithi026.stdout:589 2024-09-13T14:35:12.026 INFO:tasks.workunit.client.0.smithi026.stdout:590 2024-09-13T14:35:12.027 INFO:tasks.workunit.client.0.smithi026.stdout:591 2024-09-13T14:35:12.028 INFO:tasks.workunit.client.0.smithi026.stdout:592 2024-09-13T14:35:12.029 INFO:tasks.workunit.client.0.smithi026.stdout:593 2024-09-13T14:35:12.030 INFO:tasks.workunit.client.0.smithi026.stdout:594 2024-09-13T14:35:12.031 INFO:tasks.workunit.client.0.smithi026.stdout:595 2024-09-13T14:35:12.032 INFO:tasks.workunit.client.0.smithi026.stdout:596 2024-09-13T14:35:12.033 INFO:tasks.workunit.client.0.smithi026.stdout:597 2024-09-13T14:35:12.034 INFO:tasks.workunit.client.0.smithi026.stdout:598 2024-09-13T14:35:12.036 INFO:tasks.workunit.client.0.smithi026.stdout:599 2024-09-13T14:35:12.037 INFO:tasks.workunit.client.0.smithi026.stdout:600 2024-09-13T14:35:12.038 INFO:tasks.workunit.client.0.smithi026.stdout:601 2024-09-13T14:35:12.039 INFO:tasks.workunit.client.0.smithi026.stdout:602 2024-09-13T14:35:12.040 INFO:tasks.workunit.client.0.smithi026.stdout:603 2024-09-13T14:35:12.041 INFO:tasks.workunit.client.0.smithi026.stdout:604 2024-09-13T14:35:12.042 INFO:tasks.workunit.client.0.smithi026.stdout:605 2024-09-13T14:35:12.043 INFO:tasks.workunit.client.0.smithi026.stdout:606 2024-09-13T14:35:12.044 INFO:tasks.workunit.client.0.smithi026.stdout:607 2024-09-13T14:35:12.045 INFO:tasks.workunit.client.0.smithi026.stdout:608 2024-09-13T14:35:12.047 INFO:tasks.workunit.client.0.smithi026.stdout:609 2024-09-13T14:35:12.048 INFO:tasks.workunit.client.0.smithi026.stdout:610 2024-09-13T14:35:12.049 INFO:tasks.workunit.client.0.smithi026.stdout:611 2024-09-13T14:35:12.050 INFO:tasks.workunit.client.0.smithi026.stdout:612 2024-09-13T14:35:12.051 INFO:tasks.workunit.client.0.smithi026.stdout:613 2024-09-13T14:35:12.052 INFO:tasks.workunit.client.0.smithi026.stdout:614 2024-09-13T14:35:12.053 INFO:tasks.workunit.client.0.smithi026.stdout:615 2024-09-13T14:35:12.054 INFO:tasks.workunit.client.0.smithi026.stdout:616 2024-09-13T14:35:12.055 INFO:tasks.workunit.client.0.smithi026.stdout:617 2024-09-13T14:35:12.056 INFO:tasks.workunit.client.0.smithi026.stdout:618 2024-09-13T14:35:12.057 INFO:tasks.workunit.client.0.smithi026.stdout:619 2024-09-13T14:35:12.058 INFO:tasks.workunit.client.0.smithi026.stdout:620 2024-09-13T14:35:12.059 INFO:tasks.workunit.client.0.smithi026.stdout:621 2024-09-13T14:35:12.060 INFO:tasks.workunit.client.0.smithi026.stdout:622 2024-09-13T14:35:12.061 INFO:tasks.workunit.client.0.smithi026.stdout:623 2024-09-13T14:35:12.062 INFO:tasks.workunit.client.0.smithi026.stdout:624 2024-09-13T14:35:12.063 INFO:tasks.workunit.client.0.smithi026.stdout:625 2024-09-13T14:35:12.064 INFO:tasks.workunit.client.0.smithi026.stdout:626 2024-09-13T14:35:12.065 INFO:tasks.workunit.client.0.smithi026.stdout:627 2024-09-13T14:35:12.066 INFO:tasks.workunit.client.0.smithi026.stdout:628 2024-09-13T14:35:12.067 INFO:tasks.workunit.client.0.smithi026.stdout:629 2024-09-13T14:35:12.069 INFO:tasks.workunit.client.0.smithi026.stdout:630 2024-09-13T14:35:12.069 INFO:tasks.workunit.client.0.smithi026.stdout:631 2024-09-13T14:35:12.071 INFO:tasks.workunit.client.0.smithi026.stdout:632 2024-09-13T14:35:12.072 INFO:tasks.workunit.client.0.smithi026.stdout:633 2024-09-13T14:35:12.073 INFO:tasks.workunit.client.0.smithi026.stdout:634 2024-09-13T14:35:12.074 INFO:tasks.workunit.client.0.smithi026.stdout:635 2024-09-13T14:35:12.075 INFO:tasks.workunit.client.0.smithi026.stdout:636 2024-09-13T14:35:12.076 INFO:tasks.workunit.client.0.smithi026.stdout:637 2024-09-13T14:35:12.077 INFO:tasks.workunit.client.0.smithi026.stdout:638 2024-09-13T14:35:12.078 INFO:tasks.workunit.client.0.smithi026.stdout:639 2024-09-13T14:35:12.079 INFO:tasks.workunit.client.0.smithi026.stdout:640 2024-09-13T14:35:12.080 INFO:tasks.workunit.client.0.smithi026.stdout:641 2024-09-13T14:35:12.081 INFO:tasks.workunit.client.0.smithi026.stdout:642 2024-09-13T14:35:12.082 INFO:tasks.workunit.client.0.smithi026.stdout:643 2024-09-13T14:35:12.082 INFO:tasks.workunit.client.0.smithi026.stdout:644 2024-09-13T14:35:12.083 INFO:tasks.workunit.client.0.smithi026.stdout:645 2024-09-13T14:35:12.084 INFO:tasks.workunit.client.0.smithi026.stdout:646 2024-09-13T14:35:12.085 INFO:tasks.workunit.client.0.smithi026.stdout:647 2024-09-13T14:35:12.086 INFO:tasks.workunit.client.0.smithi026.stdout:648 2024-09-13T14:35:12.087 INFO:tasks.workunit.client.0.smithi026.stdout:649 2024-09-13T14:35:12.088 INFO:tasks.workunit.client.0.smithi026.stdout:650 2024-09-13T14:35:12.089 INFO:tasks.workunit.client.0.smithi026.stdout:651 2024-09-13T14:35:12.090 INFO:tasks.workunit.client.0.smithi026.stdout:652 2024-09-13T14:35:12.092 INFO:tasks.workunit.client.0.smithi026.stdout:653 2024-09-13T14:35:12.093 INFO:tasks.workunit.client.0.smithi026.stdout:654 2024-09-13T14:35:12.094 INFO:tasks.workunit.client.0.smithi026.stdout:655 2024-09-13T14:35:12.095 INFO:tasks.workunit.client.0.smithi026.stdout:656 2024-09-13T14:35:12.096 INFO:tasks.workunit.client.0.smithi026.stdout:657 2024-09-13T14:35:12.097 INFO:tasks.workunit.client.0.smithi026.stdout:658 2024-09-13T14:35:12.098 INFO:tasks.workunit.client.0.smithi026.stdout:659 2024-09-13T14:35:12.099 INFO:tasks.workunit.client.0.smithi026.stdout:660 2024-09-13T14:35:12.100 INFO:tasks.workunit.client.0.smithi026.stdout:661 2024-09-13T14:35:12.101 INFO:tasks.workunit.client.0.smithi026.stdout:662 2024-09-13T14:35:12.102 INFO:tasks.workunit.client.0.smithi026.stdout:663 2024-09-13T14:35:12.103 INFO:tasks.workunit.client.0.smithi026.stdout:664 2024-09-13T14:35:12.104 INFO:tasks.workunit.client.0.smithi026.stdout:665 2024-09-13T14:35:12.105 INFO:tasks.workunit.client.0.smithi026.stdout:666 2024-09-13T14:35:12.106 INFO:tasks.workunit.client.0.smithi026.stdout:667 2024-09-13T14:35:12.107 INFO:tasks.workunit.client.0.smithi026.stdout:668 2024-09-13T14:35:12.108 INFO:tasks.workunit.client.0.smithi026.stdout:669 2024-09-13T14:35:12.109 INFO:tasks.workunit.client.0.smithi026.stdout:670 2024-09-13T14:35:12.110 INFO:tasks.workunit.client.0.smithi026.stdout:671 2024-09-13T14:35:12.111 INFO:tasks.workunit.client.0.smithi026.stdout:672 2024-09-13T14:35:12.112 INFO:tasks.workunit.client.0.smithi026.stdout:673 2024-09-13T14:35:12.113 INFO:tasks.workunit.client.0.smithi026.stdout:674 2024-09-13T14:35:12.114 INFO:tasks.workunit.client.0.smithi026.stdout:675 2024-09-13T14:35:12.115 INFO:tasks.workunit.client.0.smithi026.stdout:676 2024-09-13T14:35:12.116 INFO:tasks.workunit.client.0.smithi026.stdout:677 2024-09-13T14:35:12.117 INFO:tasks.workunit.client.0.smithi026.stdout:678 2024-09-13T14:35:12.118 INFO:tasks.workunit.client.0.smithi026.stdout:679 2024-09-13T14:35:12.119 INFO:tasks.workunit.client.0.smithi026.stdout:680 2024-09-13T14:35:12.120 INFO:tasks.workunit.client.0.smithi026.stdout:681 2024-09-13T14:35:12.121 INFO:tasks.workunit.client.0.smithi026.stdout:682 2024-09-13T14:35:12.122 INFO:tasks.workunit.client.0.smithi026.stdout:683 2024-09-13T14:35:12.123 INFO:tasks.workunit.client.0.smithi026.stdout:684 2024-09-13T14:35:12.124 INFO:tasks.workunit.client.0.smithi026.stdout:685 2024-09-13T14:35:12.125 INFO:tasks.workunit.client.0.smithi026.stdout:686 2024-09-13T14:35:12.126 INFO:tasks.workunit.client.0.smithi026.stdout:687 2024-09-13T14:35:12.127 INFO:tasks.workunit.client.0.smithi026.stdout:688 2024-09-13T14:35:12.128 INFO:tasks.workunit.client.0.smithi026.stdout:689 2024-09-13T14:35:12.129 INFO:tasks.workunit.client.0.smithi026.stdout:690 2024-09-13T14:35:12.130 INFO:tasks.workunit.client.0.smithi026.stdout:691 2024-09-13T14:35:12.131 INFO:tasks.workunit.client.0.smithi026.stdout:692 2024-09-13T14:35:12.132 INFO:tasks.workunit.client.0.smithi026.stdout:693 2024-09-13T14:35:12.133 INFO:tasks.workunit.client.0.smithi026.stdout:694 2024-09-13T14:35:12.135 INFO:tasks.workunit.client.0.smithi026.stdout:695 2024-09-13T14:35:12.136 INFO:tasks.workunit.client.0.smithi026.stdout:696 2024-09-13T14:35:12.137 INFO:tasks.workunit.client.0.smithi026.stdout:697 2024-09-13T14:35:12.138 INFO:tasks.workunit.client.0.smithi026.stdout:698 2024-09-13T14:35:12.139 INFO:tasks.workunit.client.0.smithi026.stdout:699 2024-09-13T14:35:12.140 INFO:tasks.workunit.client.0.smithi026.stdout:700 2024-09-13T14:35:12.141 INFO:tasks.workunit.client.0.smithi026.stdout:701 2024-09-13T14:35:12.142 INFO:tasks.workunit.client.0.smithi026.stdout:702 2024-09-13T14:35:12.143 INFO:tasks.workunit.client.0.smithi026.stdout:703 2024-09-13T14:35:12.144 INFO:tasks.workunit.client.0.smithi026.stdout:704 2024-09-13T14:35:12.145 INFO:tasks.workunit.client.0.smithi026.stdout:705 2024-09-13T14:35:12.146 INFO:tasks.workunit.client.0.smithi026.stdout:706 2024-09-13T14:35:12.147 INFO:tasks.workunit.client.0.smithi026.stdout:707 2024-09-13T14:35:12.148 INFO:tasks.workunit.client.0.smithi026.stdout:708 2024-09-13T14:35:12.150 INFO:tasks.workunit.client.0.smithi026.stdout:709 2024-09-13T14:35:12.150 INFO:tasks.workunit.client.0.smithi026.stdout:710 2024-09-13T14:35:12.151 INFO:tasks.workunit.client.0.smithi026.stdout:711 2024-09-13T14:35:12.153 INFO:tasks.workunit.client.0.smithi026.stdout:712 2024-09-13T14:35:12.154 INFO:tasks.workunit.client.0.smithi026.stdout:713 2024-09-13T14:35:12.155 INFO:tasks.workunit.client.0.smithi026.stdout:714 2024-09-13T14:35:12.156 INFO:tasks.workunit.client.0.smithi026.stdout:715 2024-09-13T14:35:12.157 INFO:tasks.workunit.client.0.smithi026.stdout:716 2024-09-13T14:35:12.158 INFO:tasks.workunit.client.0.smithi026.stdout:717 2024-09-13T14:35:12.159 INFO:tasks.workunit.client.0.smithi026.stdout:718 2024-09-13T14:35:12.160 INFO:tasks.workunit.client.0.smithi026.stdout:719 2024-09-13T14:35:12.161 INFO:tasks.workunit.client.0.smithi026.stdout:720 2024-09-13T14:35:12.162 INFO:tasks.workunit.client.0.smithi026.stdout:721 2024-09-13T14:35:12.163 INFO:tasks.workunit.client.0.smithi026.stdout:722 2024-09-13T14:35:12.164 INFO:tasks.workunit.client.0.smithi026.stdout:723 2024-09-13T14:35:12.165 INFO:tasks.workunit.client.0.smithi026.stdout:724 2024-09-13T14:35:12.166 INFO:tasks.workunit.client.0.smithi026.stdout:725 2024-09-13T14:35:12.167 INFO:tasks.workunit.client.0.smithi026.stdout:726 2024-09-13T14:35:12.168 INFO:tasks.workunit.client.0.smithi026.stdout:727 2024-09-13T14:35:12.169 INFO:tasks.workunit.client.0.smithi026.stdout:728 2024-09-13T14:35:12.170 INFO:tasks.workunit.client.0.smithi026.stdout:729 2024-09-13T14:35:12.172 INFO:tasks.workunit.client.0.smithi026.stdout:730 2024-09-13T14:35:12.173 INFO:tasks.workunit.client.0.smithi026.stdout:731 2024-09-13T14:35:12.174 INFO:tasks.workunit.client.0.smithi026.stdout:732 2024-09-13T14:35:12.175 INFO:tasks.workunit.client.0.smithi026.stdout:733 2024-09-13T14:35:12.176 INFO:tasks.workunit.client.0.smithi026.stdout:734 2024-09-13T14:35:12.177 INFO:tasks.workunit.client.0.smithi026.stdout:735 2024-09-13T14:35:12.178 INFO:tasks.workunit.client.0.smithi026.stdout:736 2024-09-13T14:35:12.179 INFO:tasks.workunit.client.0.smithi026.stdout:737 2024-09-13T14:35:12.180 INFO:tasks.workunit.client.0.smithi026.stdout:738 2024-09-13T14:35:12.181 INFO:tasks.workunit.client.0.smithi026.stdout:739 2024-09-13T14:35:12.182 INFO:tasks.workunit.client.0.smithi026.stdout:740 2024-09-13T14:35:12.184 INFO:tasks.workunit.client.0.smithi026.stdout:741 2024-09-13T14:35:12.185 INFO:tasks.workunit.client.0.smithi026.stdout:742 2024-09-13T14:35:12.186 INFO:tasks.workunit.client.0.smithi026.stdout:743 2024-09-13T14:35:12.187 INFO:tasks.workunit.client.0.smithi026.stdout:744 2024-09-13T14:35:12.188 INFO:tasks.workunit.client.0.smithi026.stdout:745 2024-09-13T14:35:12.189 INFO:tasks.workunit.client.0.smithi026.stdout:746 2024-09-13T14:35:12.190 INFO:tasks.workunit.client.0.smithi026.stdout:747 2024-09-13T14:35:12.191 INFO:tasks.workunit.client.0.smithi026.stdout:748 2024-09-13T14:35:12.192 INFO:tasks.workunit.client.0.smithi026.stdout:749 2024-09-13T14:35:12.193 INFO:tasks.workunit.client.0.smithi026.stdout:750 2024-09-13T14:35:12.194 INFO:tasks.workunit.client.0.smithi026.stdout:751 2024-09-13T14:35:12.195 INFO:tasks.workunit.client.0.smithi026.stdout:752 2024-09-13T14:35:12.196 INFO:tasks.workunit.client.0.smithi026.stdout:753 2024-09-13T14:35:12.200 INFO:tasks.workunit.client.0.smithi026.stdout:754 2024-09-13T14:35:12.200 INFO:tasks.workunit.client.0.smithi026.stdout:755 2024-09-13T14:35:12.200 INFO:tasks.workunit.client.0.smithi026.stdout:756 2024-09-13T14:35:12.201 INFO:tasks.workunit.client.0.smithi026.stdout:757 2024-09-13T14:35:12.202 INFO:tasks.workunit.client.0.smithi026.stdout:758 2024-09-13T14:35:12.203 INFO:tasks.workunit.client.0.smithi026.stdout:759 2024-09-13T14:35:12.204 INFO:tasks.workunit.client.0.smithi026.stdout:760 2024-09-13T14:35:12.205 INFO:tasks.workunit.client.0.smithi026.stdout:761 2024-09-13T14:35:12.206 INFO:tasks.workunit.client.0.smithi026.stdout:762 2024-09-13T14:35:12.207 INFO:tasks.workunit.client.0.smithi026.stdout:763 2024-09-13T14:35:12.208 INFO:tasks.workunit.client.0.smithi026.stdout:764 2024-09-13T14:35:12.209 INFO:tasks.workunit.client.0.smithi026.stdout:765 2024-09-13T14:35:12.210 INFO:tasks.workunit.client.0.smithi026.stdout:766 2024-09-13T14:35:12.211 INFO:tasks.workunit.client.0.smithi026.stdout:767 2024-09-13T14:35:12.212 INFO:tasks.workunit.client.0.smithi026.stdout:768 2024-09-13T14:35:12.213 INFO:tasks.workunit.client.0.smithi026.stdout:769 2024-09-13T14:35:12.214 INFO:tasks.workunit.client.0.smithi026.stdout:770 2024-09-13T14:35:12.215 INFO:tasks.workunit.client.0.smithi026.stdout:771 2024-09-13T14:35:12.216 INFO:tasks.workunit.client.0.smithi026.stdout:772 2024-09-13T14:35:12.217 INFO:tasks.workunit.client.0.smithi026.stdout:773 2024-09-13T14:35:12.218 INFO:tasks.workunit.client.0.smithi026.stdout:774 2024-09-13T14:35:12.219 INFO:tasks.workunit.client.0.smithi026.stdout:775 2024-09-13T14:35:12.220 INFO:tasks.workunit.client.0.smithi026.stdout:776 2024-09-13T14:35:12.221 INFO:tasks.workunit.client.0.smithi026.stdout:777 2024-09-13T14:35:12.222 INFO:tasks.workunit.client.0.smithi026.stdout:778 2024-09-13T14:35:12.223 INFO:tasks.workunit.client.0.smithi026.stdout:779 2024-09-13T14:35:12.224 INFO:tasks.workunit.client.0.smithi026.stdout:780 2024-09-13T14:35:12.226 INFO:tasks.workunit.client.0.smithi026.stdout:781 2024-09-13T14:35:12.227 INFO:tasks.workunit.client.0.smithi026.stdout:782 2024-09-13T14:35:12.228 INFO:tasks.workunit.client.0.smithi026.stdout:783 2024-09-13T14:35:12.229 INFO:tasks.workunit.client.0.smithi026.stdout:784 2024-09-13T14:35:12.230 INFO:tasks.workunit.client.0.smithi026.stdout:785 2024-09-13T14:35:12.231 INFO:tasks.workunit.client.0.smithi026.stdout:786 2024-09-13T14:35:12.232 INFO:tasks.workunit.client.0.smithi026.stdout:787 2024-09-13T14:35:12.233 INFO:tasks.workunit.client.0.smithi026.stdout:788 2024-09-13T14:35:12.234 INFO:tasks.workunit.client.0.smithi026.stdout:789 2024-09-13T14:35:12.236 INFO:tasks.workunit.client.0.smithi026.stdout:790 2024-09-13T14:35:12.237 INFO:tasks.workunit.client.0.smithi026.stdout:791 2024-09-13T14:35:12.238 INFO:tasks.workunit.client.0.smithi026.stdout:792 2024-09-13T14:35:12.239 INFO:tasks.workunit.client.0.smithi026.stdout:793 2024-09-13T14:35:12.240 INFO:tasks.workunit.client.0.smithi026.stdout:794 2024-09-13T14:35:12.241 INFO:tasks.workunit.client.0.smithi026.stdout:795 2024-09-13T14:35:12.242 INFO:tasks.workunit.client.0.smithi026.stdout:796 2024-09-13T14:35:12.243 INFO:tasks.workunit.client.0.smithi026.stdout:797 2024-09-13T14:35:12.245 INFO:tasks.workunit.client.0.smithi026.stdout:798 2024-09-13T14:35:12.246 INFO:tasks.workunit.client.0.smithi026.stdout:799 2024-09-13T14:35:12.247 INFO:tasks.workunit.client.0.smithi026.stdout:800 2024-09-13T14:35:12.248 INFO:tasks.workunit.client.0.smithi026.stdout:801 2024-09-13T14:35:12.249 INFO:tasks.workunit.client.0.smithi026.stdout:802 2024-09-13T14:35:12.250 INFO:tasks.workunit.client.0.smithi026.stdout:803 2024-09-13T14:35:12.251 INFO:tasks.workunit.client.0.smithi026.stdout:804 2024-09-13T14:35:12.253 INFO:tasks.workunit.client.0.smithi026.stdout:805 2024-09-13T14:35:12.254 INFO:tasks.workunit.client.0.smithi026.stdout:806 2024-09-13T14:35:12.256 INFO:tasks.workunit.client.0.smithi026.stdout:807 2024-09-13T14:35:12.257 INFO:tasks.workunit.client.0.smithi026.stdout:808 2024-09-13T14:35:12.258 INFO:tasks.workunit.client.0.smithi026.stdout:809 2024-09-13T14:35:12.259 INFO:tasks.workunit.client.0.smithi026.stdout:810 2024-09-13T14:35:12.260 INFO:tasks.workunit.client.0.smithi026.stdout:811 2024-09-13T14:35:12.262 INFO:tasks.workunit.client.0.smithi026.stdout:812 2024-09-13T14:35:12.263 INFO:tasks.workunit.client.0.smithi026.stdout:813 2024-09-13T14:35:12.264 INFO:tasks.workunit.client.0.smithi026.stdout:814 2024-09-13T14:35:12.265 INFO:tasks.workunit.client.0.smithi026.stdout:815 2024-09-13T14:35:12.266 INFO:tasks.workunit.client.0.smithi026.stdout:816 2024-09-13T14:35:12.267 INFO:tasks.workunit.client.0.smithi026.stdout:817 2024-09-13T14:35:12.268 INFO:tasks.workunit.client.0.smithi026.stdout:818 2024-09-13T14:35:12.270 INFO:tasks.workunit.client.0.smithi026.stdout:819 2024-09-13T14:35:12.271 INFO:tasks.workunit.client.0.smithi026.stdout:820 2024-09-13T14:35:12.272 INFO:tasks.workunit.client.0.smithi026.stdout:821 2024-09-13T14:35:12.273 INFO:tasks.workunit.client.0.smithi026.stdout:822 2024-09-13T14:35:12.275 INFO:tasks.workunit.client.0.smithi026.stdout:823 2024-09-13T14:35:12.276 INFO:tasks.workunit.client.0.smithi026.stdout:824 2024-09-13T14:35:12.277 INFO:tasks.workunit.client.0.smithi026.stdout:825 2024-09-13T14:35:12.278 INFO:tasks.workunit.client.0.smithi026.stdout:826 2024-09-13T14:35:12.279 INFO:tasks.workunit.client.0.smithi026.stdout:827 2024-09-13T14:35:12.280 INFO:tasks.workunit.client.0.smithi026.stdout:828 2024-09-13T14:35:12.281 INFO:tasks.workunit.client.0.smithi026.stdout:829 2024-09-13T14:35:12.283 INFO:tasks.workunit.client.0.smithi026.stdout:830 2024-09-13T14:35:12.284 INFO:tasks.workunit.client.0.smithi026.stdout:831 2024-09-13T14:35:12.285 INFO:tasks.workunit.client.0.smithi026.stdout:832 2024-09-13T14:35:12.286 INFO:tasks.workunit.client.0.smithi026.stdout:833 2024-09-13T14:35:12.287 INFO:tasks.workunit.client.0.smithi026.stdout:834 2024-09-13T14:35:12.288 INFO:tasks.workunit.client.0.smithi026.stdout:835 2024-09-13T14:35:12.289 INFO:tasks.workunit.client.0.smithi026.stdout:836 2024-09-13T14:35:12.290 INFO:tasks.workunit.client.0.smithi026.stdout:837 2024-09-13T14:35:12.291 INFO:tasks.workunit.client.0.smithi026.stdout:838 2024-09-13T14:35:12.292 INFO:tasks.workunit.client.0.smithi026.stdout:839 2024-09-13T14:35:12.293 INFO:tasks.workunit.client.0.smithi026.stdout:840 2024-09-13T14:35:12.294 INFO:tasks.workunit.client.0.smithi026.stdout:841 2024-09-13T14:35:12.295 INFO:tasks.workunit.client.0.smithi026.stdout:842 2024-09-13T14:35:12.296 INFO:tasks.workunit.client.0.smithi026.stdout:843 2024-09-13T14:35:12.297 INFO:tasks.workunit.client.0.smithi026.stdout:844 2024-09-13T14:35:12.298 INFO:tasks.workunit.client.0.smithi026.stdout:845 2024-09-13T14:35:12.299 INFO:tasks.workunit.client.0.smithi026.stdout:846 2024-09-13T14:35:12.301 INFO:tasks.workunit.client.0.smithi026.stdout:847 2024-09-13T14:35:12.302 INFO:tasks.workunit.client.0.smithi026.stdout:848 2024-09-13T14:35:12.303 INFO:tasks.workunit.client.0.smithi026.stdout:849 2024-09-13T14:35:12.304 INFO:tasks.workunit.client.0.smithi026.stdout:850 2024-09-13T14:35:12.305 INFO:tasks.workunit.client.0.smithi026.stdout:851 2024-09-13T14:35:12.306 INFO:tasks.workunit.client.0.smithi026.stdout:852 2024-09-13T14:35:12.307 INFO:tasks.workunit.client.0.smithi026.stdout:853 2024-09-13T14:35:12.308 INFO:tasks.workunit.client.0.smithi026.stdout:854 2024-09-13T14:35:12.309 INFO:tasks.workunit.client.0.smithi026.stdout:855 2024-09-13T14:35:12.310 INFO:tasks.workunit.client.0.smithi026.stdout:856 2024-09-13T14:35:12.311 INFO:tasks.workunit.client.0.smithi026.stdout:857 2024-09-13T14:35:12.312 INFO:tasks.workunit.client.0.smithi026.stdout:858 2024-09-13T14:35:12.313 INFO:tasks.workunit.client.0.smithi026.stdout:859 2024-09-13T14:35:12.315 INFO:tasks.workunit.client.0.smithi026.stdout:860 2024-09-13T14:35:12.316 INFO:tasks.workunit.client.0.smithi026.stdout:861 2024-09-13T14:35:12.317 INFO:tasks.workunit.client.0.smithi026.stdout:862 2024-09-13T14:35:12.318 INFO:tasks.workunit.client.0.smithi026.stdout:863 2024-09-13T14:35:12.319 INFO:tasks.workunit.client.0.smithi026.stdout:864 2024-09-13T14:35:12.320 INFO:tasks.workunit.client.0.smithi026.stdout:865 2024-09-13T14:35:12.321 INFO:tasks.workunit.client.0.smithi026.stdout:866 2024-09-13T14:35:12.322 INFO:tasks.workunit.client.0.smithi026.stdout:867 2024-09-13T14:35:12.324 INFO:tasks.workunit.client.0.smithi026.stdout:868 2024-09-13T14:35:12.325 INFO:tasks.workunit.client.0.smithi026.stdout:869 2024-09-13T14:35:12.326 INFO:tasks.workunit.client.0.smithi026.stdout:870 2024-09-13T14:35:12.327 INFO:tasks.workunit.client.0.smithi026.stdout:871 2024-09-13T14:35:12.328 INFO:tasks.workunit.client.0.smithi026.stdout:872 2024-09-13T14:35:12.329 INFO:tasks.workunit.client.0.smithi026.stdout:873 2024-09-13T14:35:12.331 INFO:tasks.workunit.client.0.smithi026.stdout:874 2024-09-13T14:35:12.332 INFO:tasks.workunit.client.0.smithi026.stdout:875 2024-09-13T14:35:12.333 INFO:tasks.workunit.client.0.smithi026.stdout:876 2024-09-13T14:35:12.334 INFO:tasks.workunit.client.0.smithi026.stdout:877 2024-09-13T14:35:12.335 INFO:tasks.workunit.client.0.smithi026.stdout:878 2024-09-13T14:35:12.337 INFO:tasks.workunit.client.0.smithi026.stdout:879 2024-09-13T14:35:12.338 INFO:tasks.workunit.client.0.smithi026.stdout:880 2024-09-13T14:35:12.339 INFO:tasks.workunit.client.0.smithi026.stdout:881 2024-09-13T14:35:12.340 INFO:tasks.workunit.client.0.smithi026.stdout:882 2024-09-13T14:35:12.341 INFO:tasks.workunit.client.0.smithi026.stdout:883 2024-09-13T14:35:12.342 INFO:tasks.workunit.client.0.smithi026.stdout:884 2024-09-13T14:35:12.344 INFO:tasks.workunit.client.0.smithi026.stdout:885 2024-09-13T14:35:12.345 INFO:tasks.workunit.client.0.smithi026.stdout:886 2024-09-13T14:35:12.346 INFO:tasks.workunit.client.0.smithi026.stdout:887 2024-09-13T14:35:12.347 INFO:tasks.workunit.client.0.smithi026.stdout:888 2024-09-13T14:35:12.349 INFO:tasks.workunit.client.0.smithi026.stdout:889 2024-09-13T14:35:12.350 INFO:tasks.workunit.client.0.smithi026.stdout:890 2024-09-13T14:35:12.351 INFO:tasks.workunit.client.0.smithi026.stdout:891 2024-09-13T14:35:12.352 INFO:tasks.workunit.client.0.smithi026.stdout:892 2024-09-13T14:35:12.353 INFO:tasks.workunit.client.0.smithi026.stdout:893 2024-09-13T14:35:12.354 INFO:tasks.workunit.client.0.smithi026.stdout:894 2024-09-13T14:35:12.356 INFO:tasks.workunit.client.0.smithi026.stdout:895 2024-09-13T14:35:12.357 INFO:tasks.workunit.client.0.smithi026.stdout:896 2024-09-13T14:35:12.358 INFO:tasks.workunit.client.0.smithi026.stdout:897 2024-09-13T14:35:12.359 INFO:tasks.workunit.client.0.smithi026.stdout:898 2024-09-13T14:35:12.360 INFO:tasks.workunit.client.0.smithi026.stdout:899 2024-09-13T14:35:12.361 INFO:tasks.workunit.client.0.smithi026.stdout:900 2024-09-13T14:35:12.363 INFO:tasks.workunit.client.0.smithi026.stdout:901 2024-09-13T14:35:12.364 INFO:tasks.workunit.client.0.smithi026.stdout:902 2024-09-13T14:35:12.365 INFO:tasks.workunit.client.0.smithi026.stdout:903 2024-09-13T14:35:12.366 INFO:tasks.workunit.client.0.smithi026.stdout:904 2024-09-13T14:35:12.367 INFO:tasks.workunit.client.0.smithi026.stdout:905 2024-09-13T14:35:12.368 INFO:tasks.workunit.client.0.smithi026.stdout:906 2024-09-13T14:35:12.369 INFO:tasks.workunit.client.0.smithi026.stdout:907 2024-09-13T14:35:12.370 INFO:tasks.workunit.client.0.smithi026.stdout:908 2024-09-13T14:35:12.371 INFO:tasks.workunit.client.0.smithi026.stdout:909 2024-09-13T14:35:12.372 INFO:tasks.workunit.client.0.smithi026.stdout:910 2024-09-13T14:35:12.373 INFO:tasks.workunit.client.0.smithi026.stdout:911 2024-09-13T14:35:12.375 INFO:tasks.workunit.client.0.smithi026.stdout:912 2024-09-13T14:35:12.376 INFO:tasks.workunit.client.0.smithi026.stdout:913 2024-09-13T14:35:12.377 INFO:tasks.workunit.client.0.smithi026.stdout:914 2024-09-13T14:35:12.378 INFO:tasks.workunit.client.0.smithi026.stdout:915 2024-09-13T14:35:12.379 INFO:tasks.workunit.client.0.smithi026.stdout:916 2024-09-13T14:35:12.380 INFO:tasks.workunit.client.0.smithi026.stdout:917 2024-09-13T14:35:12.382 INFO:tasks.workunit.client.0.smithi026.stdout:918 2024-09-13T14:35:12.383 INFO:tasks.workunit.client.0.smithi026.stdout:919 2024-09-13T14:35:12.384 INFO:tasks.workunit.client.0.smithi026.stdout:920 2024-09-13T14:35:12.385 INFO:tasks.workunit.client.0.smithi026.stdout:921 2024-09-13T14:35:12.386 INFO:tasks.workunit.client.0.smithi026.stdout:922 2024-09-13T14:35:12.388 INFO:tasks.workunit.client.0.smithi026.stdout:923 2024-09-13T14:35:12.389 INFO:tasks.workunit.client.0.smithi026.stdout:924 2024-09-13T14:35:12.390 INFO:tasks.workunit.client.0.smithi026.stdout:925 2024-09-13T14:35:12.391 INFO:tasks.workunit.client.0.smithi026.stdout:926 2024-09-13T14:35:12.392 INFO:tasks.workunit.client.0.smithi026.stdout:927 2024-09-13T14:35:12.393 INFO:tasks.workunit.client.0.smithi026.stdout:928 2024-09-13T14:35:12.394 INFO:tasks.workunit.client.0.smithi026.stdout:929 2024-09-13T14:35:12.395 INFO:tasks.workunit.client.0.smithi026.stdout:930 2024-09-13T14:35:12.396 INFO:tasks.workunit.client.0.smithi026.stdout:931 2024-09-13T14:35:12.397 INFO:tasks.workunit.client.0.smithi026.stdout:932 2024-09-13T14:35:12.399 INFO:tasks.workunit.client.0.smithi026.stdout:933 2024-09-13T14:35:12.400 INFO:tasks.workunit.client.0.smithi026.stdout:934 2024-09-13T14:35:12.401 INFO:tasks.workunit.client.0.smithi026.stdout:935 2024-09-13T14:35:12.402 INFO:tasks.workunit.client.0.smithi026.stdout:936 2024-09-13T14:35:12.403 INFO:tasks.workunit.client.0.smithi026.stdout:937 2024-09-13T14:35:12.404 INFO:tasks.workunit.client.0.smithi026.stdout:938 2024-09-13T14:35:12.406 INFO:tasks.workunit.client.0.smithi026.stdout:939 2024-09-13T14:35:12.407 INFO:tasks.workunit.client.0.smithi026.stdout:940 2024-09-13T14:35:12.408 INFO:tasks.workunit.client.0.smithi026.stdout:941 2024-09-13T14:35:12.409 INFO:tasks.workunit.client.0.smithi026.stdout:942 2024-09-13T14:35:12.410 INFO:tasks.workunit.client.0.smithi026.stdout:943 2024-09-13T14:35:12.411 INFO:tasks.workunit.client.0.smithi026.stdout:944 2024-09-13T14:35:12.412 INFO:tasks.workunit.client.0.smithi026.stdout:945 2024-09-13T14:35:12.413 INFO:tasks.workunit.client.0.smithi026.stdout:946 2024-09-13T14:35:12.414 INFO:tasks.workunit.client.0.smithi026.stdout:947 2024-09-13T14:35:12.416 INFO:tasks.workunit.client.0.smithi026.stdout:948 2024-09-13T14:35:12.417 INFO:tasks.workunit.client.0.smithi026.stdout:949 2024-09-13T14:35:12.418 INFO:tasks.workunit.client.0.smithi026.stdout:950 2024-09-13T14:35:12.419 INFO:tasks.workunit.client.0.smithi026.stdout:951 2024-09-13T14:35:12.420 INFO:tasks.workunit.client.0.smithi026.stdout:952 2024-09-13T14:35:12.421 INFO:tasks.workunit.client.0.smithi026.stdout:953 2024-09-13T14:35:12.422 INFO:tasks.workunit.client.0.smithi026.stdout:954 2024-09-13T14:35:12.423 INFO:tasks.workunit.client.0.smithi026.stdout:955 2024-09-13T14:35:12.425 INFO:tasks.workunit.client.0.smithi026.stdout:956 2024-09-13T14:35:12.426 INFO:tasks.workunit.client.0.smithi026.stdout:957 2024-09-13T14:35:12.430 INFO:tasks.workunit.client.0.smithi026.stdout:958 2024-09-13T14:35:12.434 INFO:tasks.workunit.client.0.smithi026.stdout:959 2024-09-13T14:35:12.437 INFO:tasks.workunit.client.0.smithi026.stdout:960 2024-09-13T14:35:12.439 INFO:tasks.workunit.client.0.smithi026.stdout:961 2024-09-13T14:35:12.484 INFO:tasks.workunit.client.0.smithi026.stdout:962 2024-09-13T14:35:12.486 INFO:tasks.workunit.client.0.smithi026.stdout:963 2024-09-13T14:35:12.487 INFO:tasks.workunit.client.0.smithi026.stdout:964 2024-09-13T14:35:12.488 INFO:tasks.workunit.client.0.smithi026.stdout:965 2024-09-13T14:35:12.489 INFO:tasks.workunit.client.0.smithi026.stdout:966 2024-09-13T14:35:12.491 INFO:tasks.workunit.client.0.smithi026.stdout:967 2024-09-13T14:35:12.492 INFO:tasks.workunit.client.0.smithi026.stdout:968 2024-09-13T14:35:12.493 INFO:tasks.workunit.client.0.smithi026.stdout:969 2024-09-13T14:35:12.495 INFO:tasks.workunit.client.0.smithi026.stdout:970 2024-09-13T14:35:12.496 INFO:tasks.workunit.client.0.smithi026.stdout:971 2024-09-13T14:35:12.498 INFO:tasks.workunit.client.0.smithi026.stdout:972 2024-09-13T14:35:12.499 INFO:tasks.workunit.client.0.smithi026.stdout:973 2024-09-13T14:35:12.501 INFO:tasks.workunit.client.0.smithi026.stdout:974 2024-09-13T14:35:12.502 INFO:tasks.workunit.client.0.smithi026.stdout:975 2024-09-13T14:35:12.504 INFO:tasks.workunit.client.0.smithi026.stdout:976 2024-09-13T14:35:12.505 INFO:tasks.workunit.client.0.smithi026.stdout:977 2024-09-13T14:35:12.506 INFO:tasks.workunit.client.0.smithi026.stdout:978 2024-09-13T14:35:12.508 INFO:tasks.workunit.client.0.smithi026.stdout:979 2024-09-13T14:35:12.509 INFO:tasks.workunit.client.0.smithi026.stdout:980 2024-09-13T14:35:12.510 INFO:tasks.workunit.client.0.smithi026.stdout:981 2024-09-13T14:35:12.511 INFO:tasks.workunit.client.0.smithi026.stdout:982 2024-09-13T14:35:12.512 INFO:tasks.workunit.client.0.smithi026.stdout:983 2024-09-13T14:35:12.514 INFO:tasks.workunit.client.0.smithi026.stdout:984 2024-09-13T14:35:12.515 INFO:tasks.workunit.client.0.smithi026.stdout:985 2024-09-13T14:35:12.516 INFO:tasks.workunit.client.0.smithi026.stdout:986 2024-09-13T14:35:12.518 INFO:tasks.workunit.client.0.smithi026.stdout:987 2024-09-13T14:35:12.519 INFO:tasks.workunit.client.0.smithi026.stdout:988 2024-09-13T14:35:12.520 INFO:tasks.workunit.client.0.smithi026.stdout:989 2024-09-13T14:35:12.521 INFO:tasks.workunit.client.0.smithi026.stdout:990 2024-09-13T14:35:12.523 INFO:tasks.workunit.client.0.smithi026.stdout:991 2024-09-13T14:35:12.524 INFO:tasks.workunit.client.0.smithi026.stdout:992 2024-09-13T14:35:12.525 INFO:tasks.workunit.client.0.smithi026.stdout:993 2024-09-13T14:35:12.526 INFO:tasks.workunit.client.0.smithi026.stdout:994 2024-09-13T14:35:12.527 INFO:tasks.workunit.client.0.smithi026.stdout:995 2024-09-13T14:35:12.529 INFO:tasks.workunit.client.0.smithi026.stdout:996 2024-09-13T14:35:12.530 INFO:tasks.workunit.client.0.smithi026.stdout:997 2024-09-13T14:35:12.531 INFO:tasks.workunit.client.0.smithi026.stdout:998 2024-09-13T14:35:12.532 INFO:tasks.workunit.client.0.smithi026.stdout:999 2024-09-13T14:35:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:12 smithi145 ceph-mon[89145]: pgmap v1054: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:12 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:12 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:12 smithi145 ceph-mon[89145]: osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:35:12.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:12 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-13T14:35:12.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[99461]: pgmap v1054: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:12.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:12.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[99461]: osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[97022]: pgmap v1054: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[97022]: osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:35:12.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:12 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1082223176' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000010 69 73 |is| 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000012 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000010 69 73 |is| 2024-09-13T14:35:13.445 INFO:tasks.workunit.client.0.smithi026.stdout:00000012 2024-09-13T14:35:13.446 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.WriteReturnData (4024 ms) 2024-09-13T14:35:13.446 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.Loud 2024-09-13T14:35:13.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:13 smithi145 ceph-mon[89145]: osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:35:13.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:13 smithi026 ceph-mon[97022]: osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:35:13.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:13 smithi026 ceph-mon[99461]: osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:35:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:14 smithi145 ceph-mon[89145]: pgmap v1057: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:14 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:14 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:14.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:14 smithi145 ceph-mon[89145]: osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:35:14.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[97022]: pgmap v1057: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:14.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[97022]: osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:35:14.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[99461]: pgmap v1057: 137 pgs: 10 creating+peering, 16 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:14.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:14 smithi026 ceph-mon[99461]: osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:35:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:15 smithi145 ceph-mon[89145]: osdmap e952: 8 total, 8 up, 8 in 2024-09-13T14:35:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:15 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2568159153' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:15 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[99461]: osdmap e952: 8 total, 8 up, 8 in 2024-09-13T14:35:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2568159153' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:15.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[97022]: osdmap e952: 8 total, 8 up, 8 in 2024-09-13T14:35:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2568159153' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:15.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:15 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:16.461 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.Loud (3015 ms) 2024-09-13T14:35:16.461 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.BadMethods 2024-09-13T14:35:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[97022]: pgmap v1060: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[97022]: osdmap e953: 8 total, 8 up, 8 in 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[99461]: pgmap v1060: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:16.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:16 smithi026 ceph-mon[99461]: osdmap e953: 8 total, 8 up, 8 in 2024-09-13T14:35:16.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:16 smithi145 ceph-mon[89145]: pgmap v1060: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:16 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:16.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:16 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:16.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:16 smithi145 ceph-mon[89145]: osdmap e953: 8 total, 8 up, 8 in 2024-09-13T14:35:17.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:17 smithi026 ceph-mon[97022]: osdmap e954: 8 total, 8 up, 8 in 2024-09-13T14:35:17.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:17 smithi026 ceph-mon[99461]: osdmap e954: 8 total, 8 up, 8 in 2024-09-13T14:35:17.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:17 smithi145 ceph-mon[89145]: osdmap e954: 8 total, 8 up, 8 in 2024-09-13T14:35:18.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[99461]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[99461]: osdmap e955: 8 total, 8 up, 8 in 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[97022]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:18.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[97022]: osdmap e955: 8 total, 8 up, 8 in 2024-09-13T14:35:18.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:18.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:35:18.495+0000 7ff25b8cd640 -1 osd.3 pg_epoch: 956 pg[38.2( v 956'1 (0'0,956'1] local-lis/les=955/956 n=1 ec=955/955 lis/c=955/955 les/c/f=956/956/0 sis=955) [3,7,5] r=0 lpr=955 crt=956'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-13T14:35:18.917 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:35:18 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:35:18.496+0000 7ff2578c5640 -1 osd.3 pg_epoch: 956 pg[38.2( v 956'2 (0'0,956'2] local-lis/les=955/956 n=1 ec=955/955 lis/c=955/955 les/c/f=956/956/0 sis=955) [3,7,5] r=0 lpr=955 crt=956'1 lcod 956'1 mlcod 956'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-13T14:35:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:18 smithi145 ceph-mon[89145]: pgmap v1063: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:18 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:18 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:18 smithi145 ceph-mon[89145]: osdmap e955: 8 total, 8 up, 8 in 2024-09-13T14:35:18.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:18 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:19.487 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.BadMethods (3026 ms) 2024-09-13T14:35:19.487 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsHello.Filter 2024-09-13T14:35:19.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:19 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:19.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:19 smithi026 ceph-mon[99461]: osdmap e956: 8 total, 8 up, 8 in 2024-09-13T14:35:19.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:19 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:19.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:19 smithi026 ceph-mon[97022]: osdmap e956: 8 total, 8 up, 8 in 2024-09-13T14:35:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:19 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/200503346' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:19.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:19 smithi145 ceph-mon[89145]: osdmap e956: 8 total, 8 up, 8 in 2024-09-13T14:35:20.482 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:20.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:20 smithi026 ceph-mon[97022]: pgmap v1066: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:20.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:20 smithi026 ceph-mon[97022]: osdmap e957: 8 total, 8 up, 8 in 2024-09-13T14:35:20.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:20 smithi026 ceph-mon[99461]: pgmap v1066: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:20.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:20 smithi026 ceph-mon[99461]: osdmap e957: 8 total, 8 up, 8 in 2024-09-13T14:35:20.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:20 smithi145 ceph-mon[89145]: pgmap v1066: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:20.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:20 smithi145 ceph-mon[89145]: osdmap e957: 8 total, 8 up, 8 in 2024-09-13T14:35:21.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:21 smithi026 ceph-mon[97022]: osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:35:21.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:21 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:21.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:21] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-13T14:35:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:21 smithi026 ceph-mon[99461]: osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:35:21.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:21 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:21 smithi145 ceph-mon[89145]: osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:35:21.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:21 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:22.505 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsHello.Filter (3018 ms) 2024-09-13T14:35:22.505 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 6 tests from ClsHello (18980 ms total) 2024-09-13T14:35:22.505 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:35:22.505 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:35:22.506 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 6 tests from 1 test suite ran. (18980 ms total) 2024-09-13T14:35:22.506 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 6 tests. 2024-09-13T14:35:22.508 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:35:22.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:35:22.583 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-13T14:35:22.584 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-13T14:35:22.656 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:35:22.656 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-13T14:35:22.657 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:35:22.657 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 27 tests from TestClsJournal 2024-09-13T14:35:22.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[97022]: pgmap v1069: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:22.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[97022]: osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[99461]: pgmap v1069: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:22.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:22 smithi026 ceph-mon[99461]: osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:35:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:22 smithi145 ceph-mon[89145]: pgmap v1069: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:22 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:22 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/70577509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132689-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:22.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:22 smithi145 ceph-mon[89145]: osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:35:23.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:23 smithi026 ceph-mon[97022]: osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:35:23.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:23 smithi026 ceph-mon[99461]: osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:35:23.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:23 smithi145 ceph-mon[89145]: osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:35:24.521 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.Create 2024-09-13T14:35:24.532 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.Create (11 ms) 2024-09-13T14:35:24.532 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-13T14:35:24.536 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.MinimumSet (4 ms) 2024-09-13T14:35:24.536 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-13T14:35:24.542 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.MinimumSetStale (6 ms) 2024-09-13T14:35:24.542 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-13T14:35:24.545 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (3 ms) 2024-09-13T14:35:24.545 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-13T14:35:24.547 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ActiveSet (2 ms) 2024-09-13T14:35:24.547 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-13T14:35:24.550 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ActiveSetStale (4 ms) 2024-09-13T14:35:24.551 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-13T14:35:24.553 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.CreateDuplicate (2 ms) 2024-09-13T14:35:24.553 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-13T14:35:24.556 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.GetClient (4 ms) 2024-09-13T14:35:24.557 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-13T14:35:24.559 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientRegister (2 ms) 2024-09-13T14:35:24.559 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-13T14:35:24.563 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (4 ms) 2024-09-13T14:35:24.563 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-13T14:35:24.567 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2024-09-13T14:35:24.567 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-13T14:35:24.572 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientUpdateState (5 ms) 2024-09-13T14:35:24.572 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-13T14:35:24.575 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2024-09-13T14:35:24.575 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-13T14:35:24.579 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (4 ms) 2024-09-13T14:35:24.579 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-13T14:35:24.675 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (96 ms) 2024-09-13T14:35:24.675 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-13T14:35:24.679 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientCommit (4 ms) 2024-09-13T14:35:24.679 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-13T14:35:24.681 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (2 ms) 2024-09-13T14:35:24.681 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-13T14:35:24.682 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2024-09-13T14:35:24.682 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-13T14:35:24.719 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.ClientList (37 ms) 2024-09-13T14:35:24.719 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-13T14:35:24.723 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.GetNextTagTid (5 ms) 2024-09-13T14:35:24.723 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-13T14:35:24.732 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.TagCreate (8 ms) 2024-09-13T14:35:24.732 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-13T14:35:24.738 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (7 ms) 2024-09-13T14:35:24.739 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.TagList 2024-09-13T14:35:24.831 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.TagList (92 ms) 2024-09-13T14:35:24.831 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-13T14:35:24.833 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.GuardAppend (2 ms) 2024-09-13T14:35:24.833 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-13T14:35:24.834 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.GuardAppendDNE (1 ms) 2024-09-13T14:35:24.834 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-13T14:35:24.836 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (2 ms) 2024-09-13T14:35:24.836 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsJournal.Append 2024-09-13T14:35:24.842 INFO:tasks.workunit.client.0.smithi026.stdout:padding is not enabled 2024-09-13T14:35:24.842 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsJournal.Append (6 ms) 2024-09-13T14:35:24.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2771115097' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2771115097' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:24.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:24 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: pgmap v1072: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2771115097' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:24.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:24 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:25.527 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 27 tests from TestClsJournal (321 ms total) 2024-09-13T14:35:25.527 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:35:25.527 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:35:25.527 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 27 tests from 1 test suite ran. (2871 ms total) 2024-09-13T14:35:25.527 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 27 tests. 2024-09-13T14:35:25.529 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:35:25.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:35:25.600 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-13T14:35:25.600 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-13T14:35:25.672 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:35:25.672 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-13T14:35:25.672 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:35:25.673 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 11 tests from ClsLock 2024-09-13T14:35:25.673 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-13T14:35:25.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:25.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[97022]: osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:35:25.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[97022]: pgmap v1075: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:25.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:25.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[99461]: osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:35:25.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:25 smithi026 ceph-mon[99461]: pgmap v1075: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:25 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132829-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:25 smithi145 ceph-mon[89145]: osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:35:25.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:25 smithi145 ceph-mon[89145]: pgmap v1075: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:26.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:26 smithi026 ceph-mon[99461]: osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:35:26.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:26 smithi026 ceph-mon[97022]: osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:35:26.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:26 smithi145 ceph-mon[89145]: osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:35:26.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:27.572 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:27.572 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: exclusive 2024-09-13T14:35:27.573 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:27.573 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:27.573 INFO:tasks.workunit.client.0.smithi026.stdout: client.44738 expiration=2024-09-13T14:37:27.568834+0000 addr=v1:172.21.15.26:0/4051490007 cookie= 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: shared 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout: client.34836 expiration=0.000000 addr=v1:172.21.15.26:0/3430466792 cookie= 2024-09-13T14:35:27.577 INFO:tasks.workunit.client.0.smithi026.stdout: client.44738 expiration=2024-09-13T14:37:27.575945+0000 addr=v1:172.21.15.26:0/4051490007 cookie= 2024-09-13T14:35:27.578 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:27.578 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: shared 2024-09-13T14:35:27.578 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:27.578 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:27.578 INFO:tasks.workunit.client.0.smithi026.stdout: client.34836 expiration=0.000000 addr=v1:172.21.15.26:0/3430466792 cookie= 2024-09-13T14:35:27.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[99461]: osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:35:27.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4051490007' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[99461]: pgmap v1078: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[97022]: osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4051490007' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:27 smithi026 ceph-mon[97022]: pgmap v1078: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:27 smithi145 ceph-mon[89145]: osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:35:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:27 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4051490007' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:27 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:27.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:27 smithi145 ceph-mon[89145]: pgmap v1078: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:28.544 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestMultiLocking (2872 ms) 2024-09-13T14:35:28.544 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestMeta 2024-09-13T14:35:28.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:28.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:28.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[99461]: osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:35:28.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:28.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:28.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:28 smithi026 ceph-mon[97022]: osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:35:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:28 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:28 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:28.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:28 smithi145 ceph-mon[89145]: osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:35:29.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:29 smithi026 ceph-mon[99461]: osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:35:29.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:29 smithi026 ceph-mon[99461]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:29 smithi026 ceph-mon[97022]: osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:35:29.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:29 smithi026 ceph-mon[97022]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:29 smithi145 ceph-mon[89145]: osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:35:29.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:29 smithi145 ceph-mon[89145]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:30.560 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:30.589 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:30.589 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: shared 2024-09-13T14:35:30.590 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:30.590 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:30.590 INFO:tasks.workunit.client.0.smithi026.stdout: client.44756 expiration=0.000000 addr=v1:172.21.15.26:0/1534706420 cookie= 2024-09-13T14:35:30.591 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:30.591 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: exclusive 2024-09-13T14:35:30.591 INFO:tasks.workunit.client.0.smithi026.stdout: tag: new_tag 2024-09-13T14:35:30.592 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:30.592 INFO:tasks.workunit.client.0.smithi026.stdout: client.44750 expiration=0.000000 addr=v1:172.21.15.26:0/2246982292 cookie= 2024-09-13T14:35:30.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[99461]: osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2246982292' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[97022]: osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2246982292' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:30.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:30 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-mon[89145]: osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:35:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2246982292' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:30.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:30 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:31.575 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestMeta (3031 ms) 2024-09-13T14:35:31.575 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestCookie 2024-09-13T14:35:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[97022]: osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:35:31.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[97022]: pgmap v1084: 137 pgs: 17 creating+peering, 13 unknown, 107 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:31.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:31] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.51.0" 2024-09-13T14:35:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[99461]: osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:35:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:31 smithi026 ceph-mon[99461]: pgmap v1084: 137 pgs: 17 creating+peering, 13 unknown, 107 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:31 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:31 smithi145 ceph-mon[89145]: osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:35:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:31 smithi145 ceph-mon[89145]: pgmap v1084: 137 pgs: 17 creating+peering, 13 unknown, 107 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:32.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:32 smithi026 ceph-mon[97022]: osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:35:32.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:32 smithi026 ceph-mon[99461]: osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:35:32.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:32 smithi145 ceph-mon[89145]: osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:35:33.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[97022]: osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:35:33.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:33.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[97022]: pgmap v1087: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:33.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[99461]: osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:35:33.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:33.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:33 smithi026 ceph-mon[99461]: pgmap v1087: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:33 smithi145 ceph-mon[89145]: osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:35:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:33 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:33.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:33 smithi145 ceph-mon[89145]: pgmap v1087: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:34.064 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:34.064 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: exclusive 2024-09-13T14:35:34.064 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:34.064 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout:lock: mylock 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout: lock_type: shared 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout: tag: 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout: lockers: 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout: client.34854 expiration=0.000000 addr=v1:172.21.15.26:0/3833656367 cookie= 2024-09-13T14:35:34.067 INFO:tasks.workunit.client.0.smithi026.stdout: client.34854 expiration=0.000000 addr=v1:172.21.15.26:0/3833656367 cookie=new cookie 2024-09-13T14:35:34.587 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestCookie (3011 ms) 2024-09-13T14:35:34.587 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-13T14:35:34.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:34.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[97022]: osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:35:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:34.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:34 smithi026 ceph-mon[99461]: osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:35:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:34 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:34 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3833656367' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:34.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:34 smithi145 ceph-mon[89145]: osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:35:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:35 smithi026 ceph-mon[99461]: osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:35:35.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:35 smithi026 ceph-mon[99461]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:35.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:35 smithi026 ceph-mon[97022]: osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:35:35.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:35 smithi026 ceph-mon[97022]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:35 smithi145 ceph-mon[89145]: osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:35:35.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:35 smithi145 ceph-mon[89145]: pgmap v1090: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:36.856 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:36 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:36.857 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:36 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:36.857 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:36 smithi145 ceph-mon[89145]: osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:35:36.857 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:36 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1063938825' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:36.857 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:36 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[97022]: osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:35:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1063938825' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:36.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[99461]: osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:35:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1063938825' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:36.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:36 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:37.612 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestMultipleLocks (3025 ms) 2024-09-13T14:35:37.612 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-13T14:35:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[97022]: osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:35:37.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[97022]: pgmap v1093: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[99461]: osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:35:37.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:37 smithi026 ceph-mon[99461]: pgmap v1093: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:37.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:37 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:37.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:37 smithi145 ceph-mon[89145]: osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:35:37.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:37 smithi145 ceph-mon[89145]: pgmap v1093: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:38 smithi026 ceph-mon[97022]: osdmap e975: 8 total, 8 up, 8 in 2024-09-13T14:35:38.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:38 smithi026 ceph-mon[99461]: osdmap e975: 8 total, 8 up, 8 in 2024-09-13T14:35:38.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:38 smithi145 ceph-mon[89145]: osdmap e975: 8 total, 8 up, 8 in 2024-09-13T14:35:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[97022]: osdmap e976: 8 total, 8 up, 8 in 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/238605616' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[97022]: pgmap v1096: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[99461]: osdmap e976: 8 total, 8 up, 8 in 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/238605616' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:39 smithi026 ceph-mon[99461]: pgmap v1096: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:39 smithi145 ceph-mon[89145]: osdmap e976: 8 total, 8 up, 8 in 2024-09-13T14:35:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:39 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/238605616' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:39 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:39.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:39 smithi145 ceph-mon[89145]: pgmap v1096: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:40.618 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:40.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:40.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:40.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[97022]: osdmap e977: 8 total, 8 up, 8 in 2024-09-13T14:35:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:40.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:40 smithi026 ceph-mon[99461]: osdmap e977: 8 total, 8 up, 8 in 2024-09-13T14:35:40.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:40 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:40.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:40 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:40.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:40 smithi145 ceph-mon[89145]: osdmap e977: 8 total, 8 up, 8 in 2024-09-13T14:35:41.628 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:41] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.51.0" 2024-09-13T14:35:41.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[97022]: osdmap e978: 8 total, 8 up, 8 in 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[97022]: pgmap v1099: 137 pgs: 2 creating+peering, 13 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[99461]: osdmap e978: 8 total, 8 up, 8 in 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[99461]: pgmap v1099: 137 pgs: 2 creating+peering, 13 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:41.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:41 smithi145 ceph-mon[89145]: osdmap e978: 8 total, 8 up, 8 in 2024-09-13T14:35:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:41 smithi145 ceph-mon[89145]: pgmap v1099: 137 pgs: 2 creating+peering, 13 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:41.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:42.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:42 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:42.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:42 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:42.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:42 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:42.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:42 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:42 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:42.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:42 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:43.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:43 smithi026 ceph-mon[97022]: pgmap v1100: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:43.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:43 smithi026 ceph-mon[99461]: pgmap v1100: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:43.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:43 smithi145 ceph-mon[89145]: pgmap v1100: 137 pgs: 2 creating+peering, 6 unknown, 129 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:44.704 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestLockDuration (7092 ms) 2024-09-13T14:35:44.704 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-13T14:35:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:45 smithi145 ceph-mon[89145]: osdmap e979: 8 total, 8 up, 8 in 2024-09-13T14:35:45.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:45 smithi145 ceph-mon[89145]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:45 smithi026 ceph-mon[97022]: osdmap e979: 8 total, 8 up, 8 in 2024-09-13T14:35:46.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:45 smithi026 ceph-mon[97022]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:45 smithi026 ceph-mon[99461]: osdmap e979: 8 total, 8 up, 8 in 2024-09-13T14:35:46.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:45 smithi026 ceph-mon[99461]: pgmap v1102: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[99461]: osdmap e980: 8 total, 8 up, 8 in 2024-09-13T14:35:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3150882861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[97022]: osdmap e980: 8 total, 8 up, 8 in 2024-09-13T14:35:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3150882861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:46 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:46 smithi145 ceph-mon[89145]: osdmap e980: 8 total, 8 up, 8 in 2024-09-13T14:35:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:46 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3150882861' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:46 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:47.725 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestAssertLocked (3021 ms) 2024-09-13T14:35:47.725 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-13T14:35:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[97022]: osdmap e981: 8 total, 8 up, 8 in 2024-09-13T14:35:48.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[97022]: pgmap v1105: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[99461]: osdmap e981: 8 total, 8 up, 8 in 2024-09-13T14:35:48.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:47 smithi026 ceph-mon[99461]: pgmap v1105: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:47 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:47 smithi145 ceph-mon[89145]: osdmap e981: 8 total, 8 up, 8 in 2024-09-13T14:35:48.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:47 smithi145 ceph-mon[89145]: pgmap v1105: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:49.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:48 smithi026 ceph-mon[97022]: osdmap e982: 8 total, 8 up, 8 in 2024-09-13T14:35:49.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:48 smithi026 ceph-mon[99461]: osdmap e982: 8 total, 8 up, 8 in 2024-09-13T14:35:49.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:48 smithi145 ceph-mon[89145]: osdmap e982: 8 total, 8 up, 8 in 2024-09-13T14:35:50.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[97022]: osdmap e983: 8 total, 8 up, 8 in 2024-09-13T14:35:50.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:50.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[97022]: pgmap v1108: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:50.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[99461]: osdmap e983: 8 total, 8 up, 8 in 2024-09-13T14:35:50.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:50.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:49 smithi026 ceph-mon[99461]: pgmap v1108: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:49 smithi145 ceph-mon[89145]: osdmap e983: 8 total, 8 up, 8 in 2024-09-13T14:35:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:49 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:50.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:49 smithi145 ceph-mon[89145]: pgmap v1108: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:35:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:35:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:50.743 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestSetCookie (3017 ms) 2024-09-13T14:35:50.743 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestRenew 2024-09-13T14:35:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:51.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:51.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[97022]: osdmap e984: 8 total, 8 up, 8 in 2024-09-13T14:35:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:51.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:50 smithi026 ceph-mon[99461]: osdmap e984: 8 total, 8 up, 8 in 2024-09-13T14:35:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:50 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:50 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2460866560' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:51.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:50 smithi145 ceph-mon[89145]: osdmap e984: 8 total, 8 up, 8 in 2024-09-13T14:35:51.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:35:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:35:51] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.51.0" 2024-09-13T14:35:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:51 smithi026 ceph-mon[97022]: osdmap e985: 8 total, 8 up, 8 in 2024-09-13T14:35:52.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:51 smithi026 ceph-mon[97022]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:51 smithi026 ceph-mon[99461]: osdmap e985: 8 total, 8 up, 8 in 2024-09-13T14:35:52.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:51 smithi026 ceph-mon[99461]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:52.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:51 smithi145 ceph-mon[89145]: osdmap e985: 8 total, 8 up, 8 in 2024-09-13T14:35:52.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:51 smithi145 ceph-mon[89145]: pgmap v1111: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:53.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[97022]: osdmap e986: 8 total, 8 up, 8 in 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/762826849' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[99461]: osdmap e986: 8 total, 8 up, 8 in 2024-09-13T14:35:53.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/762826849' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:53.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:52 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:52 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:52 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:35:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:52 smithi145 ceph-mon[89145]: osdmap e986: 8 total, 8 up, 8 in 2024-09-13T14:35:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:52 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/762826849' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:53.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:52 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:54.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[99461]: osdmap e987: 8 total, 8 up, 8 in 2024-09-13T14:35:54.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[99461]: pgmap v1114: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[97022]: osdmap e987: 8 total, 8 up, 8 in 2024-09-13T14:35:54.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:53 smithi026 ceph-mon[97022]: pgmap v1114: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:53 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:53 smithi145 ceph-mon[89145]: osdmap e987: 8 total, 8 up, 8 in 2024-09-13T14:35:54.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:53 smithi145 ceph-mon[89145]: pgmap v1114: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:55.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:54 smithi026 ceph-mon[97022]: osdmap e988: 8 total, 8 up, 8 in 2024-09-13T14:35:55.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:54 smithi026 ceph-mon[99461]: osdmap e988: 8 total, 8 up, 8 in 2024-09-13T14:35:55.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:54 smithi145 ceph-mon[89145]: osdmap e988: 8 total, 8 up, 8 in 2024-09-13T14:35:56.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:55 smithi026 ceph-mon[97022]: pgmap v1116: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:56.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:55 smithi026 ceph-mon[99461]: pgmap v1116: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:56.205 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:55 smithi145 ceph-mon[89145]: pgmap v1116: 137 pgs: 3 creating+peering, 20 unknown, 114 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:57.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:57 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:57.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:57.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:57 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:57.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:35:57.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:57 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:58.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:35:58 smithi145 ceph-mon[89145]: pgmap v1117: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:58.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:35:58 smithi026 ceph-mon[97022]: pgmap v1117: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:58.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:35:58 smithi026 ceph-mon[99461]: pgmap v1117: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:00.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:00 smithi145 ceph-mon[89145]: pgmap v1118: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 143 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:00.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:00.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:00 smithi026 ceph-mon[97022]: pgmap v1118: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 143 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:00.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:00 smithi026 ceph-mon[99461]: pgmap v1118: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 143 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:01.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:01] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.51.0" 2024-09-13T14:36:02.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:02 smithi145 ceph-mon[89145]: pgmap v1119: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 125 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:02.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:02 smithi026 ceph-mon[97022]: pgmap v1119: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 125 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:02.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:02 smithi026 ceph-mon[99461]: pgmap v1119: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 125 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:03.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:03.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:03.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:03.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:04.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:04 smithi145 ceph-mon[89145]: pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 307 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:04.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:04 smithi026 ceph-mon[97022]: pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 307 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:04.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:04 smithi026 ceph-mon[99461]: pgmap v1120: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 307 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:06.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:06 smithi145 ceph-mon[89145]: pgmap v1121: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 275 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:06.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:06 smithi026 ceph-mon[97022]: pgmap v1121: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 275 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:06.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:06 smithi026 ceph-mon[99461]: pgmap v1121: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 275 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:08.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:08 smithi145 ceph-mon[89145]: pgmap v1122: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:08 smithi026 ceph-mon[97022]: pgmap v1122: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:08 smithi026 ceph-mon[99461]: pgmap v1122: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:10.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:10 smithi145 ceph-mon[89145]: pgmap v1123: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:10.456 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:10.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:10 smithi026 ceph-mon[97022]: pgmap v1123: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:10.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:10 smithi026 ceph-mon[99461]: pgmap v1123: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-13T14:36:11.208 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestRenew (20465 ms) 2024-09-13T14:36:11.208 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-13T14:36:11.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:11.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:11.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:11.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:11] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-09-13T14:36:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:12 smithi145 ceph-mon[89145]: pgmap v1124: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:12.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:12 smithi145 ceph-mon[89145]: osdmap e989: 8 total, 8 up, 8 in 2024-09-13T14:36:12.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:12 smithi026 ceph-mon[99461]: pgmap v1124: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:12.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:12 smithi026 ceph-mon[99461]: osdmap e989: 8 total, 8 up, 8 in 2024-09-13T14:36:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:12 smithi026 ceph-mon[97022]: pgmap v1124: 137 pgs: 137 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:12.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:12 smithi026 ceph-mon[97022]: osdmap e989: 8 total, 8 up, 8 in 2024-09-13T14:36:13.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:13 smithi026 ceph-mon[97022]: osdmap e990: 8 total, 8 up, 8 in 2024-09-13T14:36:13.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:13 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:13.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:13 smithi026 ceph-mon[99461]: osdmap e990: 8 total, 8 up, 8 in 2024-09-13T14:36:13.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:13 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:13.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:13 smithi145 ceph-mon[89145]: osdmap e990: 8 total, 8 up, 8 in 2024-09-13T14:36:13.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:13 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[97022]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[97022]: osdmap e991: 8 total, 8 up, 8 in 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[99461]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:14 smithi026 ceph-mon[99461]: osdmap e991: 8 total, 8 up, 8 in 2024-09-13T14:36:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:14 smithi145 ceph-mon[89145]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:14 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:14 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1414189989' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:14 smithi145 ceph-mon[89145]: osdmap e991: 8 total, 8 up, 8 in 2024-09-13T14:36:15.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:15 smithi026 ceph-mon[99461]: osdmap e992: 8 total, 8 up, 8 in 2024-09-13T14:36:15.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:15 smithi026 ceph-mon[97022]: osdmap e992: 8 total, 8 up, 8 in 2024-09-13T14:36:15.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:15 smithi145 ceph-mon[89145]: osdmap e992: 8 total, 8 up, 8 in 2024-09-13T14:36:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[97022]: pgmap v1130: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:16.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:36:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[99461]: pgmap v1130: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:16.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:16 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:36:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:16 smithi145 ceph-mon[89145]: pgmap v1130: 137 pgs: 3 creating+peering, 23 unknown, 111 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:16 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:16.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:16 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:36:18.237 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7028 ms) 2024-09-13T14:36:18.237 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-13T14:36:18.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:18 smithi026 ceph-mon[97022]: pgmap v1131: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:18.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:18 smithi026 ceph-mon[99461]: pgmap v1131: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:18.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:18 smithi145 ceph-mon[89145]: pgmap v1131: 137 pgs: 3 creating+peering, 1 unknown, 133 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:19.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:19 smithi026 ceph-mon[97022]: osdmap e993: 8 total, 8 up, 8 in 2024-09-13T14:36:19.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:19 smithi026 ceph-mon[99461]: osdmap e993: 8 total, 8 up, 8 in 2024-09-13T14:36:19.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:19 smithi145 ceph-mon[89145]: osdmap e993: 8 total, 8 up, 8 in 2024-09-13T14:36:20.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[99461]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[99461]: osdmap e994: 8 total, 8 up, 8 in 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4009035586' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[97022]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[97022]: osdmap e994: 8 total, 8 up, 8 in 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4009035586' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:20 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:20 smithi145 ceph-mon[89145]: pgmap v1133: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:20 smithi145 ceph-mon[89145]: osdmap e994: 8 total, 8 up, 8 in 2024-09-13T14:36:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:20 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4009035586' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:20 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:20.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:21.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:21 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:21.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:21 smithi026 ceph-mon[99461]: osdmap e995: 8 total, 8 up, 8 in 2024-09-13T14:36:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:21 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:21.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:21 smithi026 ceph-mon[97022]: osdmap e995: 8 total, 8 up, 8 in 2024-09-13T14:36:21.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:21] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-09-13T14:36:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:21 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:21.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:21 smithi145 ceph-mon[89145]: osdmap e995: 8 total, 8 up, 8 in 2024-09-13T14:36:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:22 smithi026 ceph-mon[97022]: pgmap v1136: 137 pgs: 17 creating+peering, 11 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:22.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:22 smithi026 ceph-mon[97022]: osdmap e996: 8 total, 8 up, 8 in 2024-09-13T14:36:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:22 smithi026 ceph-mon[99461]: pgmap v1136: 137 pgs: 17 creating+peering, 11 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:22.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:22 smithi026 ceph-mon[99461]: osdmap e996: 8 total, 8 up, 8 in 2024-09-13T14:36:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:22 smithi145 ceph-mon[89145]: pgmap v1136: 137 pgs: 17 creating+peering, 11 unknown, 109 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:22.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:22 smithi145 ceph-mon[89145]: osdmap e996: 8 total, 8 up, 8 in 2024-09-13T14:36:24.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:24 smithi026 ceph-mon[97022]: pgmap v1138: 137 pgs: 17 creating+peering, 8 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:24.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:24 smithi026 ceph-mon[99461]: pgmap v1138: 137 pgs: 17 creating+peering, 8 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:24.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:24 smithi145 ceph-mon[89145]: pgmap v1138: 137 pgs: 17 creating+peering, 8 unknown, 112 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:26.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8042 ms) 2024-09-13T14:36:26.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-13T14:36:26.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:26 smithi026 ceph-mon[97022]: pgmap v1139: 137 pgs: 17 creating+peering, 120 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:26.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:26 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:26 smithi026 ceph-mon[99461]: pgmap v1139: 137 pgs: 17 creating+peering, 120 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:26.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:26 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:26 smithi145 ceph-mon[89145]: pgmap v1139: 137 pgs: 17 creating+peering, 120 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:26.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:26 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:27.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:27 smithi026 ceph-mon[97022]: osdmap e997: 8 total, 8 up, 8 in 2024-09-13T14:36:27.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:27 smithi026 ceph-mon[99461]: osdmap e997: 8 total, 8 up, 8 in 2024-09-13T14:36:27.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:27 smithi145 ceph-mon[89145]: osdmap e997: 8 total, 8 up, 8 in 2024-09-13T14:36:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[97022]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:28.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[97022]: osdmap e998: 8 total, 8 up, 8 in 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/3140411013' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[99461]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[99461]: osdmap e998: 8 total, 8 up, 8 in 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/3140411013' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:28.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:28 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:28 smithi145 ceph-mon[89145]: pgmap v1141: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:28 smithi145 ceph-mon[89145]: osdmap e998: 8 total, 8 up, 8 in 2024-09-13T14:36:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:28 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/3140411013' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:28.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:28 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:29 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:29.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:29 smithi026 ceph-mon[97022]: osdmap e999: 8 total, 8 up, 8 in 2024-09-13T14:36:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:29 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:29.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:29 smithi026 ceph-mon[99461]: osdmap e999: 8 total, 8 up, 8 in 2024-09-13T14:36:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:29 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-132884-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:29.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:29 smithi145 ceph-mon[89145]: osdmap e999: 8 total, 8 up, 8 in 2024-09-13T14:36:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:30 smithi026 ceph-mon[97022]: pgmap v1144: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:30.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:30 smithi026 ceph-mon[97022]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-13T14:36:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:30 smithi026 ceph-mon[99461]: pgmap v1144: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:30.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:30 smithi026 ceph-mon[99461]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-13T14:36:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:30 smithi145 ceph-mon[89145]: pgmap v1144: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:30.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:30 smithi145 ceph-mon[89145]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-13T14:36:30.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:31.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:31] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.51.0" 2024-09-13T14:36:32.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:32 smithi026 ceph-mon[97022]: pgmap v1146: 137 pgs: 7 creating+peering, 8 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:32.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:32 smithi026 ceph-mon[99461]: pgmap v1146: 137 pgs: 7 creating+peering, 8 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:32.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:32 smithi145 ceph-mon[89145]: pgmap v1146: 137 pgs: 7 creating+peering, 8 unknown, 122 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8032 ms) 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 11 tests from ClsLock (68639 ms total) 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 11 tests from 1 test suite ran. (68639 ms total) 2024-09-13T14:36:34.311 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 11 tests. 2024-09-13T14:36:34.315 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:36:34.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:36:34.387 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-13T14:36:34.387 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-13T14:36:34.461 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:36:34.462 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-13T14:36:34.462 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:36:34.462 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 4 tests from cls_log 2024-09-13T14:36:34.462 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-13T14:36:34.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:34 smithi026 ceph-mon[97022]: pgmap v1147: 137 pgs: 7 creating+peering, 130 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:34.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:34 smithi026 ceph-mon[99461]: pgmap v1147: 137 pgs: 7 creating+peering, 130 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:34.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:34 smithi145 ceph-mon[89145]: pgmap v1147: 137 pgs: 7 creating+peering, 130 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:35.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:35 smithi026 ceph-mon[97022]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-13T14:36:35.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:35 smithi026 ceph-mon[99461]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-13T14:36:35.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:35 smithi145 ceph-mon[89145]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:36 smithi145 ceph-mon[89145]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:36 smithi145 ceph-mon[89145]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:36 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:36.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[97022]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[97022]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-13T14:36:36.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[99461]: pgmap v1149: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[99461]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-13T14:36:36.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:36 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:37.332 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_log.test_log_add_same_time (2871 ms) 2024-09-13T14:36:37.332 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-13T14:36:37.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:37 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:37.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:37 smithi026 ceph-mon[97022]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-13T14:36:37.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:37 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:37.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:37 smithi026 ceph-mon[99461]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-13T14:36:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:37 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/289751325' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:37.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:37 smithi145 ceph-mon[89145]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-13T14:36:38.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:38 smithi026 ceph-mon[99461]: pgmap v1152: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:38.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:38 smithi026 ceph-mon[99461]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-13T14:36:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:38 smithi026 ceph-mon[97022]: pgmap v1152: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:38.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:38 smithi026 ceph-mon[97022]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-13T14:36:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:38 smithi145 ceph-mon[89145]: pgmap v1152: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:38.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:38 smithi145 ceph-mon[89145]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-13T14:36:39.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:39 smithi026 ceph-mon[97022]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:36:39.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:39 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:39.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:39 smithi026 ceph-mon[99461]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:36:39.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:39 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:39 smithi145 ceph-mon[89145]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:36:39.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:39 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:40.348 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_log.test_log_add_different_time (3015 ms) 2024-09-13T14:36:40.348 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_log.trim_by_time 2024-09-13T14:36:40.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[97022]: pgmap v1155: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[97022]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[99461]: pgmap v1155: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:40.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:40 smithi026 ceph-mon[99461]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:36:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:40 smithi145 ceph-mon[89145]: pgmap v1155: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:40 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:40 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1979051551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:40.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:40 smithi145 ceph-mon[89145]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:36:40.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[97022]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:36:41.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:41.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[99461]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:36:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:41.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:41.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:41 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:41] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-09-13T14:36:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:41 smithi145 ceph-mon[89145]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:36:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:41.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:41 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[97022]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:42.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[97022]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[99461]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[99461]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:36:42.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:42 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:42 smithi145 ceph-mon[89145]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:42 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:42 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:36:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:42 smithi145 ceph-mon[89145]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:36:42.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:42 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:43.375 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_log.trim_by_time (3028 ms) 2024-09-13T14:36:43.375 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-13T14:36:43.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:43 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:43.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:43 smithi026 ceph-mon[97022]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:36:43.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:43 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:43.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:43 smithi026 ceph-mon[99461]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:36:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:43 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4283263689' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:43.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:43 smithi145 ceph-mon[89145]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:36:44.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:44 smithi026 ceph-mon[99461]: pgmap v1161: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:44.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:44 smithi026 ceph-mon[99461]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:36:44.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:44 smithi026 ceph-mon[97022]: pgmap v1161: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:44.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:44 smithi026 ceph-mon[97022]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:36:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:44 smithi145 ceph-mon[89145]: pgmap v1161: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:44.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:44 smithi145 ceph-mon[89145]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:36:45.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[97022]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:36:45.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1373058801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:45.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:45.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[99461]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:36:45.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1373058801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:45.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:45 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:45 smithi145 ceph-mon[89145]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:36:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:45 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1373058801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:45.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:45 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:46.395 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] cls_log.trim_by_marker (3020 ms) 2024-09-13T14:36:46.395 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 4 tests from cls_log (11934 ms total) 2024-09-13T14:36:46.395 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:36:46.395 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:36:46.396 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 4 tests from 1 test suite ran. (11934 ms total) 2024-09-13T14:36:46.396 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 4 tests. 2024-09-13T14:36:46.397 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:36:46.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:36:46.469 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-13T14:36:46.469 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-13T14:36:46.542 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:36:46.542 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-13T14:36:46.542 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:36:46.542 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 4 tests from ClsNumOps 2024-09-13T14:36:46.542 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsNumOps.Add 2024-09-13T14:36:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[97022]: pgmap v1164: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:46.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:46.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[97022]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:36:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[99461]: pgmap v1164: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:46.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:46 smithi026 ceph-mon[99461]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:36:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:46 smithi145 ceph-mon[89145]: pgmap v1164: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:46 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:46 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133270-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:46.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:46 smithi145 ceph-mon[89145]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:36:47.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:47 smithi026 ceph-mon[99461]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:36:47.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:47 smithi026 ceph-mon[97022]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:36:47.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:47 smithi145 ceph-mon[89145]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:36:48.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/445920755' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/445920755' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:48.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:48 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: pgmap v1167: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/445920755' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:48.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:48 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:49.414 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsNumOps.Add (2872 ms) 2024-09-13T14:36:49.414 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsNumOps.Sub 2024-09-13T14:36:49.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:49 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:49.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:49 smithi026 ceph-mon[99461]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:36:49.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:49 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:49.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:49 smithi026 ceph-mon[97022]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:36:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:49 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:49.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:49 smithi145 ceph-mon[89145]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:36:50.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:50 smithi026 ceph-mon[99461]: pgmap v1170: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:50.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:50 smithi026 ceph-mon[99461]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:36:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:50 smithi026 ceph-mon[97022]: pgmap v1170: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:50.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:50 smithi026 ceph-mon[97022]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:36:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:50 smithi145 ceph-mon[89145]: pgmap v1170: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:50.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:50 smithi145 ceph-mon[89145]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:36:50.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:36:50 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:36:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:51.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[99461]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1617206519' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[97022]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1617206519' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:36:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:36:51] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-09-13T14:36:51.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:51 smithi145 ceph-mon[89145]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:36:51.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:51 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1617206519' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:51 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:52.432 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsNumOps.Sub (3018 ms) 2024-09-13T14:36:52.433 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsNumOps.Mul 2024-09-13T14:36:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:52 smithi145 ceph-mon[89145]: pgmap v1173: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:52.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:52 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:52.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:52 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:52.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:52 smithi145 ceph-mon[89145]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:36:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[97022]: pgmap v1173: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:52.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:52.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[97022]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:36:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[99461]: pgmap v1173: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:52.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:52 smithi026 ceph-mon[99461]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:36:53.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:53 smithi145 ceph-mon[89145]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:36:53.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:53 smithi026 ceph-mon[97022]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:36:53.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:53 smithi026 ceph-mon[99461]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:36:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:54 smithi145 ceph-mon[89145]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:54 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:54 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:36:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:54 smithi145 ceph-mon[89145]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:36:54.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:54 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[99461]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:36:54.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[99461]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[97022]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[97022]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:36:54.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:54 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.446 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsNumOps.Mul (3014 ms) 2024-09-13T14:36:55.447 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] ClsNumOps.Div 2024-09-13T14:36:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:55 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:55.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:55 smithi145 ceph-mon[89145]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:36:55.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:55 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:55.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:55 smithi026 ceph-mon[99461]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:36:55.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:55 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4051188515' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:55.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:55 smithi026 ceph-mon[97022]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:36:56.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:56 smithi145 ceph-mon[89145]: pgmap v1179: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:56.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:56 smithi145 ceph-mon[89145]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:36:56.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:56.706 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:56 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[97022]: pgmap v1179: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[97022]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:36:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:56.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[99461]: pgmap v1179: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[99461]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:36:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:36:56.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:56 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:57.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:57 smithi145 ceph-mon[89145]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:36:57.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:57 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/598276931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:57.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:57 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:57.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[97022]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:36:57.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/598276931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:57.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[99461]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:36:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/598276931' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:57.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:57 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:58.468 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] ClsNumOps.Div (3022 ms) 2024-09-13T14:36:58.469 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 4 tests from ClsNumOps (11926 ms total) 2024-09-13T14:36:58.469 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:36:58.469 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:36:58.469 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 4 tests from 1 test suite ran. (11926 ms total) 2024-09-13T14:36:58.469 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 4 tests. 2024-09-13T14:36:58.471 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:36:58.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:36:58.545 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-13T14:36:58.545 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-13T14:36:58.617 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2024-09-13T14:36:58.617 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-13T14:36:58.617 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2024-09-13T14:36:58.617 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd 2024-09-13T14:36:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:58 smithi145 ceph-mon[89145]: pgmap v1182: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:58 smithi145 ceph-mon[89145]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:58 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:58.705 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:58 smithi145 ceph-mon[89145]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:36:58.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[99461]: pgmap v1182: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:58.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[99461]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:58.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:58.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[99461]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:36:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[97022]: pgmap v1182: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:36:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[97022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133376-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:58.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:58 smithi026 ceph-mon[97022]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:36:59.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:36:59 smithi026 ceph-mon[99461]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:36:59.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:36:59 smithi026 ceph-mon[97022]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:36:59.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:36:59 smithi145 ceph-mon[89145]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:37:00.468 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:37:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:00.479 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-13T14:37:00.493 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_all_features (14 ms) 2024-09-13T14:37:00.493 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.copyup 2024-09-13T14:37:00.685 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.copyup (192 ms) 2024-09-13T14:37:00.685 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-13T14:37:00.700 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparse_copyup (15 ms) 2024-09-13T14:37:00.700 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-13T14:37:00.707 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-13T14:37:00.707 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-13T14:37:00.718 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2024-09-13T14:37:00.718 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-13T14:37:00.744 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2024-09-13T14:37:00.744 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.create 2024-09-13T14:37:00.757 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.create (13 ms) 2024-09-13T14:37:00.757 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_features 2024-09-13T14:37:00.758 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-13T14:37:00.759 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-13T14:37:00.760 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-13T14:37:00.760 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-13T14:37:00.762 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-13T14:37:00.762 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-13T14:37:00.764 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-13T14:37:00.764 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-13T14:37:00.765 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-13T14:37:00.765 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-13T14:37:00.769 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-09-13T14:37:00.769 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_size 2024-09-13T14:37:00.774 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2024-09-13T14:37:00.774 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_size 2024-09-13T14:37:00.778 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-13T14:37:00.778 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-13T14:37:00.794 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.protection_status (16 ms) 2024-09-13T14:37:00.794 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-13T14:37:00.801 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-13T14:37:00.801 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-13T14:37:00.841 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v1 (40 ms) 2024-09-13T14:37:00.841 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-13T14:37:00.870 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-13T14:37:00.870 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-13T14:37:00.891 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-09-13T14:37:00.891 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-13T14:37:00.898 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2024-09-13T14:37:00.898 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-13T14:37:00.907 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2024-09-13T14:37:00.908 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-13T14:37:00.909 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-09-13T14:37:00.909 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/1570572878' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/1570572878' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:00 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.916 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-09-13T14:37:00.917 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-13T14:37:00.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2024-09-13T14:37:00.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-13T14:37:00.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-13T14:37:00.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-13T14:37:00.928 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2024-09-13T14:37:00.929 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-13T14:37:00.935 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_remove (6 ms) 2024-09-13T14:37:00.935 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.flags 2024-09-13T14:37:00.939 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-13T14:37:00.939 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.metadata 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: pgmap v1185: 105 pgs: 105 active+clean; 583 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/1570572878' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:00 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:00.963 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2024-09-13T14:37:00.963 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_features 2024-09-13T14:37:00.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-13T14:37:00.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror 2024-09-13T14:37:01.002 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2024-09-13T14:37:01.002 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-13T14:37:01.017 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2024-09-13T14:37:01.017 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-13T14:37:01.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:01 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:01.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:01 smithi026 ceph-mon[99461]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:37:01.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:01 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:01.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:01 smithi026 ceph-mon[97022]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:37:01.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:01 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:37:01] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.51.0" 2024-09-13T14:37:01.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:01 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-133482-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:01.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:01 smithi145 ceph-mon[89145]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:37:02.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:02 smithi026 ceph-mon[99461]: pgmap v1188: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:02.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:02 smithi026 ceph-mon[99461]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:37:02.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:02 smithi026 ceph-mon[97022]: pgmap v1188: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:02.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:02 smithi026 ceph-mon[97022]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:37:02.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:02 smithi145 ceph-mon[89145]: pgmap v1188: 137 pgs: 10 creating+peering, 15 unknown, 112 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:02.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:02 smithi145 ceph-mon[89145]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:37:03.409 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_status (2392 ms) 2024-09-13T14:37:03.410 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-13T14:37:03.493 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_map (84 ms) 2024-09-13T14:37:03.493 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-13T14:37:03.500 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-13T14:37:03.501 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-13T14:37:03.512 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-13T14:37:03.512 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-13T14:37:03.514 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-13T14:37:03.514 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-13T14:37:03.516 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-13T14:37:03.516 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-13T14:37:03.519 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-13T14:37:03.519 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-13T14:37:03.524 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-13T14:37:03.525 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-13T14:37:03.528 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-13T14:37:03.528 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-13T14:37:03.532 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-13T14:37:03.532 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-13T14:37:03.534 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-13T14:37:03.534 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-13T14:37:03.537 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-13T14:37:03.537 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-13T14:37:03.539 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-13T14:37:03.539 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-13T14:37:03.544 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-13T14:37:03.544 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-13T14:37:03.546 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-13T14:37:03.546 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-13T14:37:03.549 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-13T14:37:03.549 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-13T14:37:03.552 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2024-09-13T14:37:03.552 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-13T14:37:03.554 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-13T14:37:03.554 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-13T14:37:03.556 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-13T14:37:03.556 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-13T14:37:03.558 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-13T14:37:03.559 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-13T14:37:03.561 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-09-13T14:37:03.561 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-13T14:37:03.563 INFO:tasks.workunit.client.0.smithi026.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1627-gbbdadfa3/rpm/el9/BUILD/ceph-17.2.7-1627-gbbdadfa3/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-13T14:37:03.563 INFO:tasks.workunit.client.0.smithi026.stdout:Expected equality of these values: 2024-09-13T14:37:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: 1U 2024-09-13T14:37:03.563 INFO:tasks.workunit.client.0.smithi026.stdout: Which is: 1 2024-09-13T14:37:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: keys.size() 2024-09-13T14:37:03.564 INFO:tasks.workunit.client.0.smithi026.stdout: Which is: 3 2024-09-13T14:37:03.564 INFO:tasks.workunit.client.0.smithi026.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-13T14:37:03.564 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-13T14:37:03.566 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-09-13T14:37:03.567 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-13T14:37:03.582 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-13T14:37:03.582 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-13T14:37:03.780 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (197 ms) 2024-09-13T14:37:03.780 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1627-gbbdadfa3/rpm/el9/BUILD/ceph-17.2.7-1627-gbbdadfa3/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout:Expected equality of these values: 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: 1U 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: Which is: 1 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: keys.size() 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout: Which is: 3 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout:[ FAILED ] TestClsRbd.group_snap_remove (2 ms) 2024-09-13T14:37:03.782 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-13T14:37:03.784 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-13T14:37:03.784 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-13T14:37:03.791 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-13T14:37:03.791 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.op_features 2024-09-13T14:37:03.798 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-13T14:37:03.798 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-13T14:37:03.819 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2024-09-13T14:37:03.819 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-13T14:37:03.827 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2024-09-13T14:37:03.827 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-13T14:37:03.839 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-09-13T14:37:03.839 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-13T14:37:03.846 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-13T14:37:03.846 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration 2024-09-13T14:37:03.856 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration (10 ms) 2024-09-13T14:37:03.856 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-13T14:37:03.865 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2024-09-13T14:37:03.865 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-13T14:37:03.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:03.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:03.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:03.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:37:03.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:03.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:03.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:03.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:03 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:37:03.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:03.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:03.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:03.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:03 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' 2024-09-13T14:37:04.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:04 smithi026 ceph-mon[99461]: pgmap v1190: 137 pgs: 10 creating+peering, 4 unknown, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 54 KiB/s rd, 17 KiB/s wr, 60 op/s 2024-09-13T14:37:04.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:04 smithi026 ceph-mon[97022]: pgmap v1190: 137 pgs: 10 creating+peering, 4 unknown, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 54 KiB/s rd, 17 KiB/s wr, 60 op/s 2024-09-13T14:37:04.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:04 smithi145 ceph-mon[89145]: pgmap v1190: 137 pgs: 10 creating+peering, 4 unknown, 123 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 54 KiB/s rd, 17 KiB/s wr, 60 op/s 2024-09-13T14:37:05.502 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1637 ms) 2024-09-13T14:37:05.502 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-13T14:37:05.541 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparsify (39 ms) 2024-09-13T14:37:05.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:05 smithi026 ceph-mon[97022]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:37:05.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:05 smithi026 ceph-mon[99461]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:37:05.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:05 smithi145 ceph-mon[89145]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:37:06.509 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd (5062 ms total) 2024-09-13T14:37:06.509 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:37:06.509 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2024-09-13T14:37:06.509 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 70 tests from 1 test suite ran. (7892 ms total) 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 68 tests. 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout:[ FAILED ] 2 tests, listed below: 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout: 2024-09-13T14:37:06.510 INFO:tasks.workunit.client.0.smithi026.stdout: 2 FAILED TESTS 2024-09-13T14:37:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.512 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-13T14:37:06.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:37:06.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:06 smithi026 ceph-mon[99461]: pgmap v1192: 137 pgs: 10 creating+peering, 127 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 17 KiB/s wr, 58 op/s 2024-09-13T14:37:06.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:06 smithi026 ceph-mon[99461]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:37:06.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:06 smithi026 ceph-mon[97022]: pgmap v1192: 137 pgs: 10 creating+peering, 127 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 17 KiB/s wr, 58 op/s 2024-09-13T14:37:06.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:06 smithi026 ceph-mon[97022]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:37:06.795 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi026 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T14:37:07.045 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=5b34c29ea8d54cee9bb0cafdc5cdc28c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi026 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T14:37:07.048 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T14:37:07.057 INFO:tasks.cephadm:Teardown begin 2024-09-13T14:37:07.058 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-13T14:37:07.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:37:07.071 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:06 smithi145 ceph-mon[89145]: pgmap v1192: 137 pgs: 10 creating+peering, 127 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 50 KiB/s rd, 17 KiB/s wr, 58 op/s 2024-09-13T14:37:07.072 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:06 smithi145 ceph-mon[89145]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:37:07.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:07.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:07 smithi026 ceph-mon[99461]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:37:07.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:07 smithi026 ceph-mon[97022]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:37:07.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:07 smithi145 ceph-mon[89145]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:37:08.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:08 smithi026 ceph-mon[97022]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:08.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:08 smithi026 ceph-mon[99461]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:08.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:08 smithi145 ceph-mon[89145]: pgmap v1195: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:09.444 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:37:09.444 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":16,"btime":"2024-09-13T14:28:27:995468+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.145:6833/1802307086","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":1802307086},{"type":"v1","addr":"172.21.15.145:6833","nonce":1802307086}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":16}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:28:20.369282+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":598,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44408},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44408":{"gid":44408,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":6,"addr":"172.21.15.26:6835/3923891201","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":3923891201},{"type":"v1","addr":"172.21.15.26:6835","nonce":3923891201}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44408,"qdb_cluster":[44408]},"id":1}]} 2024-09-13T14:37:09.444 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 16 2024-09-13T14:37:09.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:09 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2495055024' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:09.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:09 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2495055024' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:09.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:09 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2495055024' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:10.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-13T14:37:10.516 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:37:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:37:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:10.550 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:10.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:10 smithi026 ceph-mon[99461]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:10.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:10 smithi026 ceph-mon[97022]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:10.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:10 smithi145 ceph-mon[89145]: pgmap v1196: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:11.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:11 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:37:11] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-09-13T14:37:11.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:11 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:11.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:11 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:11.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:11 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:12.652 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:37:12.652 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":16,"btime":"2024-09-13T14:28:27:995468+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.145:6833/1802307086","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":1802307086},{"type":"v1","addr":"172.21.15.145:6833","nonce":1802307086}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":16}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-13T14:15:28.665431+0000","modified":"2024-09-13T14:28:20.369282+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":598,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44408},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44408":{"gid":44408,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":6,"addr":"172.21.15.26:6835/3923891201","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6834","nonce":3923891201},{"type":"v1","addr":"172.21.15.26:6835","nonce":3923891201}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44408,"qdb_cluster":[44408]},"id":1}]} 2024-09-13T14:37:12.653 INFO:teuthology.orchestra.run.smithi026.stderr:dumped fsmap epoch 16 2024-09-13T14:37:12.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:12 smithi026 ceph-mon[97022]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:12.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:12 smithi026 ceph-mon[99461]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:12.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:12 smithi145 ceph-mon[89145]: pgmap v1197: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:13.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:13 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/75301371' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:13.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:13 smithi026 ceph-mon[99461]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:13.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:13 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/75301371' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:13.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:13 smithi026 ceph-mon[97022]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:13.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:37:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:13 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/75301371' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-13T14:37:13.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:13 smithi145 ceph-mon[89145]: pgmap v1198: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:14.040 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:15.989 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-09-13T14:37:15.989 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":1031,"fsid":"ad455616-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:09:53.788860+0000","modified":"2024-09-13T14:37:06.496360+0000","last_up_change":"2024-09-13T14:27:54.980682+0000","last_in_change":"2024-09-13T14:14:39.052925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":60,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:12:55.698911+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-13T14:15:21.478645+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-13T14:15:25.002865+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-13T14:16:19.892063+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ff918df3-c46e-4975-9452-aa5b2025532d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":332,"up_from":346,"up_thru":1026,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6803","nonce":3754976775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6805","nonce":3754976775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6809","nonce":3754976775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3754976775},{"type":"v1","addr":"172.21.15.26:6807","nonce":3754976775}]},"public_addr":"172.21.15.26:6803/3754976775","cluster_addr":"172.21.15.26:6805/3754976775","heartbeat_back_addr":"172.21.15.26:6809/3754976775","heartbeat_front_addr":"172.21.15.26:6807/3754976775","state":["exists","up"]},{"osd":1,"uuid":"247b896e-5e0a-47ef-b8c6-61618b71477a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":366,"up_from":378,"up_thru":1026,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6811","nonce":2254834333}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6813","nonce":2254834333}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6817","nonce":2254834333}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2254834333},{"type":"v1","addr":"172.21.15.26:6815","nonce":2254834333}]},"public_addr":"172.21.15.26:6811/2254834333","cluster_addr":"172.21.15.26:6813/2254834333","heartbeat_back_addr":"172.21.15.26:6817/2254834333","heartbeat_front_addr":"172.21.15.26:6815/2254834333","state":["exists","up"]},{"osd":2,"uuid":"cbb32835-8bfe-413c-86dc-b733e8800551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":399,"up_from":412,"up_thru":1026,"down_at":400,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6819","nonce":718889581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6821","nonce":718889581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6825","nonce":718889581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":718889581},{"type":"v1","addr":"172.21.15.26:6823","nonce":718889581}]},"public_addr":"172.21.15.26:6819/718889581","cluster_addr":"172.21.15.26:6821/718889581","heartbeat_back_addr":"172.21.15.26:6825/718889581","heartbeat_front_addr":"172.21.15.26:6823/718889581","state":["exists","up"]},{"osd":3,"uuid":"f1fa5de3-4d3b-4555-ba58-e4ec6a10369d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":430,"up_from":443,"up_thru":1026,"down_at":431,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6827","nonce":3139338605}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6829","nonce":3139338605}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6833","nonce":3139338605}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3139338605},{"type":"v1","addr":"172.21.15.26:6831","nonce":3139338605}]},"public_addr":"172.21.15.26:6827/3139338605","cluster_addr":"172.21.15.26:6829/3139338605","heartbeat_back_addr":"172.21.15.26:6833/3139338605","heartbeat_front_addr":"172.21.15.26:6831/3139338605","state":["exists","up"]},{"osd":4,"uuid":"c0c8497e-75fb-4238-9c1e-c1593ecf2507","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":463,"up_from":474,"up_thru":1026,"down_at":464,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6801","nonce":3668564396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6803","nonce":3668564396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6807","nonce":3668564396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":3668564396},{"type":"v1","addr":"172.21.15.145:6805","nonce":3668564396}]},"public_addr":"172.21.15.145:6801/3668564396","cluster_addr":"172.21.15.145:6803/3668564396","heartbeat_back_addr":"172.21.15.145:6807/3668564396","heartbeat_front_addr":"172.21.15.145:6805/3668564396","state":["exists","up"]},{"osd":5,"uuid":"aa1efee7-55a1-4905-a6ca-b3c54deb71bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":495,"up_from":508,"up_thru":1026,"down_at":496,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6809","nonce":313394676}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6811","nonce":313394676}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6815","nonce":313394676}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":313394676},{"type":"v1","addr":"172.21.15.145:6813","nonce":313394676}]},"public_addr":"172.21.15.145:6809/313394676","cluster_addr":"172.21.15.145:6811/313394676","heartbeat_back_addr":"172.21.15.145:6815/313394676","heartbeat_front_addr":"172.21.15.145:6813/313394676","state":["exists","up"]},{"osd":6,"uuid":"79482f04-e0b2-494c-8d25-ab2ff68d191c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":529,"up_from":543,"up_thru":1026,"down_at":530,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6817","nonce":3319036697}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6819","nonce":3319036697}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6823","nonce":3319036697}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3319036697},{"type":"v1","addr":"172.21.15.145:6821","nonce":3319036697}]},"public_addr":"172.21.15.145:6817/3319036697","cluster_addr":"172.21.15.145:6819/3319036697","heartbeat_back_addr":"172.21.15.145:6823/3319036697","heartbeat_front_addr":"172.21.15.145:6821/3319036697","state":["exists","up"]},{"osd":7,"uuid":"6f406363-f9fd-4e98-b256-0fb5b4ffaf60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":563,"up_from":576,"up_thru":1026,"down_at":564,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6825","nonce":2839525764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6827","nonce":2839525764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6831","nonce":2839525764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2839525764},{"type":"v1","addr":"172.21.15.145:6829","nonce":2839525764}]},"public_addr":"172.21.15.145:6825/2839525764","cluster_addr":"172.21.15.145:6827/2839525764","heartbeat_back_addr":"172.21.15.145:6831/2839525764","heartbeat_front_addr":"172.21.15.145:6829/2839525764","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:04.191735+0000","dead_epoch":332},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:29.206650+0000","dead_epoch":366},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:12:53.239169+0000","dead_epoch":398},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:19.035459+0000","dead_epoch":430},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:40.953006+0000","dead_epoch":463},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:05.325025+0000","dead_epoch":495},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:29.504548+0000","dead_epoch":529},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:54.446292+0000","dead_epoch":563}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"3.1","mappings":[{"from":4,"to":1}]},{"pgid":"3.6","mappings":[{"from":0,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.30","mappings":[{"from":0,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6833/2983197144":"2024-09-14T14:28:17.341368+0000","172.21.15.145:6832/2983197144":"2024-09-14T14:28:17.341368+0000","172.21.15.26:0/1320392898":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/3486505461":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/3971773587":"2024-09-14T14:22:40.678958+0000","172.21.15.26:6801/2523901053":"2024-09-14T14:22:40.678958+0000","172.21.15.26:6800/2523901053":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/2708858480":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/602611566":"2024-09-14T14:22:40.678958+0000","172.21.15.145:6835/1781482491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:6801/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/4257981059":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/2247322222":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/3661859927":"2024-09-14T14:10:21.076248+0000","172.21.15.145:0/1853142526":"2024-09-14T14:21:59.347678+0000","172.21.15.145:0/203034452":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/3794283059":"2024-09-14T14:10:45.203536+0000","172.21.15.145:0/1500293491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2245841676":"2024-09-14T14:20:46.093511+0000","172.21.15.26:6801/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:6801/295355582":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/969696743":"2024-09-14T14:10:45.203536+0000","172.21.15.26:0/4132567630":"2024-09-14T14:22:40.678958+0000","172.21.15.26:0/1842776456":"2024-09-14T14:16:17.348500+0000","172.21.15.145:0/2745661972":"2024-09-14T14:21:59.347678+0000","172.21.15.145:0/3573890460":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2209085404":"2024-09-14T14:20:46.093511+0000","172.21.15.145:6834/1781482491":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/3526758430":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/3442380687":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/173360804":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6800/1838587817":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/2134193589":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/3350103151":"2024-09-14T14:10:21.076248+0000","172.21.15.26:0/2881487463":"2024-09-14T14:20:46.093511+0000","172.21.15.26:6800/295355582":"2024-09-14T14:20:46.093511+0000","172.21.15.145:0/93878221":"2024-09-14T14:21:59.347678+0000","172.21.15.26:0/2881792973":"2024-09-14T14:20:46.093511+0000","172.21.15.26:0/4259195615":"2024-09-14T14:10:45.203536+0000","172.21.15.26:6800/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:6801/724813798":"2024-09-14T14:16:17.348500+0000","172.21.15.26:0/2605336872":"2024-09-14T14:22:40.678958+0000","172.21.15.145:0/2960544754":"2024-09-14T14:21:59.347678+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":471,"length":1}]},{"pool":60,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:37:16.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:15 smithi026 ceph-mon[97022]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:16.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:15 smithi026 ceph-mon[99461]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:16.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:15 smithi145 ceph-mon[89145]: pgmap v1199: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:16.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs fail cephfs 2024-09-13T14:37:17.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:16 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2686068598' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:37:17.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:16 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2686068598' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:37:17.176 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:17.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:16 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2686068598' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:37:18.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:17 smithi026 ceph-mon[99461]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:18.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:17 smithi026 ceph-mon[97022]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:18.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:17 smithi145 ceph-mon[89145]: pgmap v1200: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:19.110 INFO:teuthology.orchestra.run.smithi026.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-13T14:37:19.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[96999]: 2024-09-13T14:37:19.107+0000 7fe1fa64b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:37:19.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: -4408> 2024-09-13T14:28:22.258+0000 7fa9ce433640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-13T14:37:19.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: ignoring --setuser ceph since I am not root 2024-09-13T14:37:19.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: ignoring --setgroup ceph since I am not root 2024-09-13T14:37:19.416 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: starting mds.a at 2024-09-13T14:37:19.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-09-13T14:37:20.098 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:20.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:20.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[99461]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:37:20.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:37:20.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:19 smithi026 ceph-mon[97022]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:37:20.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:37:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:37:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:20.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/4175837498' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:37:20.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:19 smithi145 ceph-mon[89145]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-13T14:37:21.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[99461]: mds.? [v2:172.21.15.26:6834/3022558797,v1:172.21.15.26:6835/3022558797] up:boot 2024-09-13T14:37:21.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[99461]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-13T14:37:21.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:37:21.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[97022]: mds.? [v2:172.21.15.26:6834/3022558797,v1:172.21.15.26:6835/3022558797] up:boot 2024-09-13T14:37:21.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[97022]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-13T14:37:21.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:37:21.456 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:21 smithi145 ceph-mon[89145]: mds.? [v2:172.21.15.26:6834/3022558797,v1:172.21.15.26:6835/3022558797] up:boot 2024-09-13T14:37:21.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:21 smithi145 ceph-mon[89145]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-13T14:37:21.457 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:21 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-13T14:37:21.657 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:21 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:37:21] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-09-13T14:37:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[97022]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/426761163' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[99461]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:22.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/426761163' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:22 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:22 smithi145 ceph-mon[89145]: pgmap v1203: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:22 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/426761163' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:22 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:22.730 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-13T14:37:22.990 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:23.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[97022]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:37:23.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[97022]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:37:23.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[97022]: Cluster is now healthy 2024-09-13T14:37:23.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:23.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[97022]: fsmap 2 up:standby 2024-09-13T14:37:23.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[99461]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:37:23.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[99461]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:37:23.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[99461]: Cluster is now healthy 2024-09-13T14:37:23.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:23.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:23 smithi026 ceph-mon[99461]: fsmap 2 up:standby 2024-09-13T14:37:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:23 smithi145 ceph-mon[89145]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:37:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:23 smithi145 ceph-mon[89145]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:37:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:23 smithi145 ceph-mon[89145]: Cluster is now healthy 2024-09-13T14:37:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:23 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:23.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:23 smithi145 ceph-mon[89145]: fsmap 2 up:standby 2024-09-13T14:37:24.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:24 smithi026 ceph-mon[99461]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:24.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:24 smithi026 ceph-mon[97022]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:24.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:24 smithi145 ceph-mon[89145]: pgmap v1204: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:25.133 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'cephfs_metadata' removed 2024-09-13T14:37:25.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:25 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:25.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:25 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:25.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:25 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:25.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ad455616-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-13T14:37:26.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:26.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[97022]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:26.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:26.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[97022]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:37:26.393 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[99461]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:26.393 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:26.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:26 smithi026 ceph-mon[99461]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:37:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:26 smithi145 ceph-mon[89145]: pgmap v1205: 105 pgs: 105 active+clean; 583 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:26 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/2587341253' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:26.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:26 smithi145 ceph-mon[89145]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:37:27.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:27 smithi026 ceph-mon[99461]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:27.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:27 smithi026 ceph-mon[97022]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:27.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:27 smithi145 ceph-mon[89145]: from='mgr.25142 172.21.15.26:0/2159947506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:28.146 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'cephfs_data' removed 2024-09-13T14:37:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[97022]: pgmap v1207: 73 pgs: 73 active+clean; 577 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:28.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[97022]: from='client.? 172.21.15.26:0/169909086' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[99461]: pgmap v1207: 73 pgs: 73 active+clean; 577 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[99461]: from='client.? 172.21.15.26:0/169909086' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:28 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:28 smithi145 ceph-mon[89145]: pgmap v1207: 73 pgs: 73 active+clean; 577 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:37:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:28 smithi145 ceph-mon[89145]: from='client.? 172.21.15.26:0/169909086' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:28 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:37:28.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:37:28.865 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:37:28.901 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T14:37:28.901 DEBUG:teuthology.orchestra.run.smithi026:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:37:28.922 DEBUG:teuthology.orchestra.run.smithi145:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:37:28.958 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T14:37:28.958 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T14:37:28.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:37:29.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 systemd[1]: Stopping Ceph mon.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:29.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-mon[99461]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:29.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-mon[99461]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-13T14:37:29.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE Bus STOPPING 2024-09-13T14:37:29.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:37:29.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE Bus STOPPED 2024-09-13T14:37:29.416 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE Bus STARTING 2024-09-13T14:37:29.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-mon[97022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:29.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-mon[97022]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-13T14:37:29.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[96999]: 2024-09-13T14:37:29.290+0000 7fe1ffe56640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:29.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a[96999]: 2024-09-13T14:37:29.290+0000 7fe1ffe56640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T14:37:29.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 podman[135204]: 2024-09-13 14:37:29.383652554 +0000 UTC m=+0.227034465 container died fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:37:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:29 smithi145 ceph-mon[89145]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:37:29.455 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:29 smithi145 ceph-mon[89145]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-13T14:37:29.749 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE Serving on http://:::9283 2024-09-13T14:37:29.749 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:29 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: [13/Sep/2024:14:37:29] ENGINE Bus STARTED 2024-09-13T14:37:30.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 podman[135204]: 2024-09-13 14:37:29.74856311 +0000 UTC m=+0.591945005 container cleanup fde66f55593e6602eb1583c2477bea27d301720a24d72b764708467dbdd989af (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:37:30.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 13 14:37:29 smithi026 bash[135204]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:37:30.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T14:37:30.249 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:30.249 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T14:37:30.250 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T14:37:30.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c 2024-09-13T14:37:30.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:30 smithi026 systemd[1]: Stopping Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:30.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:37:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:37:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:30.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c[99437]: 2024-09-13T14:37:30.553+0000 7fc24825b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:30.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:30 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c[99437]: 2024-09-13T14:37:30.553+0000 7fc24825b640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T14:37:30.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:30 smithi026 podman[135347]: 2024-09-13 14:37:30.801290791 +0000 UTC m=+0.353931762 container died f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T14:37:31.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 podman[135347]: 2024-09-13 14:37:31.186666207 +0000 UTC m=+0.739307177 container cleanup f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:37:31.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 bash[135347]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:37:31.397 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:31 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: ::ffff:172.21.15.145 - - [13/Sep/2024:14:37:31] "GET /metrics HTTP/1.1" 200 31605 "" "Prometheus/2.51.0" 2024-09-13T14:37:31.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T14:37:31.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 podman[135365]: 2024-09-13 14:37:31.396499145 +0000 UTC m=+0.590610246 container remove f85c39c40108dfc5017fa0547a00e959bddb465bcea7c4db14f75ab02c6546bc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:37:31.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:37:31.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 systemd[1]: Stopped Ceph mon.c for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:37:31.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 13 14:37:31 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.c.service: Consumed 12.412s CPU time. 2024-09-13T14:37:31.677 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:31.677 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T14:37:31.677 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T14:37:31.678 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.b 2024-09-13T14:37:31.955 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:31 smithi145 systemd[1]: Stopping Ceph mon.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:32.453 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b[89122]: 2024-09-13T14:37:31.977+0000 7f2719e36640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:32.453 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b[89122]: 2024-09-13T14:37:31.977+0000 7f2719e36640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T14:37:32.453 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:32 smithi145 podman[124050]: 2024-09-13 14:37:32.203015523 +0000 UTC m=+0.334382760 container died 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T14:37:32.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:32 smithi145 podman[124050]: 2024-09-13 14:37:32.660184169 +0000 UTC m=+0.791551403 container cleanup 22caa9373f03f007d6d98105a60bbf4e3ba99e9452e97748f3bf71952ced4e44 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:32.708 INFO:journalctl@ceph.mon.b.smithi145.stdout:Sep 13 14:37:32 smithi145 bash[124050]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:37:33.144 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T14:37:33.200 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:33.201 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T14:37:33.201 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T14:37:33.201 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:37:33.536 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 systemd[1]: Stopping Ceph mgr.y for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:33.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:37:33.533+0000 7f833be76640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:33.859 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y[88147]: 2024-09-13T14:37:33.533+0000 7f833be76640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-13T14:37:33.860 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 podman[135479]: 2024-09-13 14:37:33.566802424 +0000 UTC m=+0.168730222 container died 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:37:34.165 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 podman[135479]: 2024-09-13 14:37:33.949706838 +0000 UTC m=+0.551634677 container cleanup 1443c0c3060c47a372ce64336d53b5fdb5a9e7f01aad81a88e53a715dcffd6f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:37:34.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 13 14:37:33 smithi026 bash[135479]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:37:34.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T14:37:34.427 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:34.427 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T14:37:34.428 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T14:37:34.428 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x 2024-09-13T14:37:34.765 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:37:34 smithi145 systemd[1]: Stopping Ceph mgr.x for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:35.058 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:37:34 smithi145 podman[124183]: 2024-09-13 14:37:34.764888414 +0000 UTC m=+0.133339847 container died 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:35.392 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:37:35 smithi145 podman[124183]: 2024-09-13 14:37:35.14041811 +0000 UTC m=+0.508869546 container cleanup 6f4dd502efd77717ca87d6fe83dd5782e07e9db2bf2b28ae82a57923e7379dda (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:37:35.392 INFO:journalctl@ceph.mgr.x.smithi145.stdout:Sep 13 14:37:35 smithi145 bash[124183]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:37:35.528 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T14:37:35.563 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:35.563 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T14:37:35.564 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T14:37:35.564 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.0 2024-09-13T14:37:35.864 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:35 smithi026 systemd[1]: Stopping Ceph osd.0 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:36.165 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:37:35.863+0000 7f9eda8fc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:36.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:37:35.863+0000 7f9eda8fc640 -1 osd.0 1034 *** Got signal Terminated *** 2024-09-13T14:37:36.166 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:35 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0[105168]: 2024-09-13T14:37:35.863+0000 7f9eda8fc640 -1 osd.0 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:37:40.705 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:37:40 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:37:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ad455616-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:41.556 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:41 smithi026 podman[135611]: 2024-09-13 14:37:41.274746933 +0000 UTC m=+5.513714930 container died 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2) 2024-09-13T14:37:41.914 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:41 smithi026 podman[135611]: 2024-09-13 14:37:41.662680052 +0000 UTC m=+5.901648038 container cleanup 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-09-13T14:37:41.915 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:41 smithi026 bash[135611]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:37:42.165 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:41 smithi026 podman[135631]: 2024-09-13 14:37:41.914128786 +0000 UTC m=+0.627671771 container remove 3429eb79404f666ac6ebbb7c014acb5e1eff550eb5c813405b10fc4f2b12c726 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:37:42.517 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.139651338 +0000 UTC m=+0.016769893 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:42.518 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.290075965 +0000 UTC m=+0.167194561 container create ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:42.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.517529972 +0000 UTC m=+0.394648538 container init ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T14:37:42.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.522777329 +0000 UTC m=+0.399895887 container start ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T14:37:42.916 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.608945386 +0000 UTC m=+0.486063952 container attach ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T14:37:42.917 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:42 smithi026 podman[135730]: 2024-09-13 14:37:42.7164868 +0000 UTC m=+0.593605355 container died ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:43.257 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 13 14:37:43 smithi026 podman[135773]: 2024-09-13 14:37:43.254354234 +0000 UTC m=+0.532151900 container remove ffa7cfcd1bfa1326e24f2adb81c7dbb5ac673c460330c2aba2ff27ba005be38d (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T14:37:43.277 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T14:37:43.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:43.322 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T14:37:43.322 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T14:37:43.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.1 2024-09-13T14:37:43.665 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:43 smithi026 systemd[1]: Stopping Ceph osd.1 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:44.165 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:37:43.667+0000 7f85f7cd0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:44.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:37:43.667+0000 7f85f7cd0640 -1 osd.1 1034 *** Got signal Terminated *** 2024-09-13T14:37:44.166 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:43 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1[110386]: 2024-09-13T14:37:43.667+0000 7f85f7cd0640 -1 osd.1 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:37:49.542 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:49 smithi026 podman[135859]: 2024-09-13 14:37:49.227404873 +0000 UTC m=+5.667839203 container died 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T14:37:49.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:49 smithi026 podman[135859]: 2024-09-13 14:37:49.740481764 +0000 UTC m=+6.180916096 container cleanup 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T14:37:49.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:49 smithi026 bash[135859]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:37:50.292 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:49 smithi026 podman[135880]: 2024-09-13 14:37:49.966975913 +0000 UTC m=+0.732515879 container remove 2441dc634f7cfb01a3f43a1e981b58fda90a4a6fadc4015999e001e6650c7afa (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T14:37:50.612 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.194493747 +0000 UTC m=+0.016294317 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:50.612 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.301110454 +0000 UTC m=+0.122911013 container create 6bf369ce51b4707c6795e29fac57114d8f752451bbe8c5fe6c885da15df075bd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T14:37:50.612 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.512166407 +0000 UTC m=+0.333966971 container init 6bf369ce51b4707c6795e29fac57114d8f752451bbe8c5fe6c885da15df075bd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True) 2024-09-13T14:37:50.613 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.517633316 +0000 UTC m=+0.339433878 container start 6bf369ce51b4707c6795e29fac57114d8f752451bbe8c5fe6c885da15df075bd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T14:37:50.613 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.611785264 +0000 UTC m=+0.433585825 container attach 6bf369ce51b4707c6795e29fac57114d8f752451bbe8c5fe6c885da15df075bd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:50.915 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 13 14:37:50 smithi026 podman[135974]: 2024-09-13 14:37:50.714749478 +0000 UTC m=+0.536550043 container died 6bf369ce51b4707c6795e29fac57114d8f752451bbe8c5fe6c885da15df075bd (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:37:51.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T14:37:51.443 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:51.444 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T14:37:51.444 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T14:37:51.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.2 2024-09-13T14:37:51.665 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:51 smithi026 systemd[1]: Stopping Ceph osd.2 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:52.165 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:37:51.803+0000 7f4527d81640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:37:52.166 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:37:51.803+0000 7f4527d81640 -1 osd.2 1034 *** Got signal Terminated *** 2024-09-13T14:37:52.166 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2[115362]: 2024-09-13T14:37:51.803+0000 7f4527d81640 -1 osd.2 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:37:57.612 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:57 smithi026 podman[136103]: 2024-09-13 14:37:57.355726802 +0000 UTC m=+5.682788005 container died 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:37:57.915 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:57 smithi026 podman[136103]: 2024-09-13 14:37:57.777250205 +0000 UTC m=+6.104311407 container cleanup 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:37:57.915 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:57 smithi026 bash[136103]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:37:58.405 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:57 smithi026 podman[136135]: 2024-09-13 14:37:57.970286035 +0000 UTC m=+0.611848803 container remove 581bbbb517ea2548d4c9d526376bda699d395981ecaac6f8dad925a6e50912e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:37:58.665 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.404453961 +0000 UTC m=+0.112189488 container create 2ac01bbab8e84c90d2cc9807d68bfe48991d1b59e0a10e9cb928b21ce2211afe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:37:58.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.308280785 +0000 UTC m=+0.016016267 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:58.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.615246776 +0000 UTC m=+0.322982249 container init 2ac01bbab8e84c90d2cc9807d68bfe48991d1b59e0a10e9cb928b21ce2211afe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:37:58.666 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.620308528 +0000 UTC m=+0.328044003 container start 2ac01bbab8e84c90d2cc9807d68bfe48991d1b59e0a10e9cb928b21ce2211afe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:37:59.093 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.723273778 +0000 UTC m=+0.431009254 container attach 2ac01bbab8e84c90d2cc9807d68bfe48991d1b59e0a10e9cb928b21ce2211afe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:37:59.094 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 13 14:37:58 smithi026 podman[136227]: 2024-09-13 14:37:58.811262999 +0000 UTC m=+0.518998472 container died 2ac01bbab8e84c90d2cc9807d68bfe48991d1b59e0a10e9cb928b21ce2211afe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T14:37:59.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T14:37:59.460 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:37:59.460 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T14:37:59.460 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T14:37:59.460 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3 2024-09-13T14:37:59.665 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:37:59 smithi026 systemd[1]: Stopping Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:00.165 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:37:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:37:59.814+0000 7ff2732fa640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:00.165 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:37:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:37:59.814+0000 7ff2732fa640 -1 osd.3 1034 *** Got signal Terminated *** 2024-09-13T14:38:00.166 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:37:59 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:37:59.814+0000 7ff2732fa640 -1 osd.3 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:02.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:02 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:38:02.161+0000 7ff26f90f640 -1 osd.3 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:36.902271+0000 front 2024-09-13T14:37:36.902324+0000 (oldest deadline 2024-09-13T14:38:01.602282+0000) 2024-09-13T14:38:02.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:01 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:01.978+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:03.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:02.751+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:03.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:02 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:02.952+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:03.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:03 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:38:03.139+0000 7ff26f90f640 -1 osd.3 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:36.902271+0000 front 2024-09-13T14:37:36.902324+0000 (oldest deadline 2024-09-13T14:38:01.602282+0000) 2024-09-13T14:38:04.027 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:03.944+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:04.027 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:03 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:03.765+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:04.440 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:04.026+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:04.665 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:04 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3[120321]: 2024-09-13T14:38:04.181+0000 7ff26f90f640 -1 osd.3 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:36.902271+0000 front 2024-09-13T14:37:36.902324+0000 (oldest deadline 2024-09-13T14:38:01.602282+0000) 2024-09-13T14:38:04.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:04.439+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:05.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:04.812+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:05.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:04.928+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:05.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:04 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:04.997+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:05.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:05 smithi026 podman[136354]: 2024-09-13 14:38:05.161662505 +0000 UTC m=+5.475268454 container died 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T14:38:05.780 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:05 smithi026 podman[136354]: 2024-09-13 14:38:05.521191592 +0000 UTC m=+5.834797545 container cleanup 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:38:05.781 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:05 smithi026 bash[136354]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:38:05.838 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:05.457+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:06.098 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:05 smithi026 podman[136375]: 2024-09-13 14:38:05.780553352 +0000 UTC m=+0.613173690 container remove 2146eb0c6926dcb2a8e551793a98fc4aa702b70284f02f7adaf91abdb225bde5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T14:38:06.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:05.962+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:06.206 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:05.837+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:06.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:05 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:05.963+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:06.415 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.097871027 +0000 UTC m=+0.112789585 container create 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:38:06.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.001100539 +0000 UTC m=+0.016019098 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:06.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.333829015 +0000 UTC m=+0.348747572 container init 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:38:06.416 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.338827459 +0000 UTC m=+0.353746016 container start 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T14:38:06.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:06.424+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:06.820 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.430757217 +0000 UTC m=+0.445675790 container attach 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:38:06.821 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:06 smithi026 podman[136468]: 2024-09-13 14:38:06.534268382 +0000 UTC m=+0.549186941 container died 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T14:38:07.067 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T14:38:07.091 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:07 smithi026 podman[136512]: 2024-09-13 14:38:07.045319226 +0000 UTC m=+0.505281424 container remove 0a3b366bab756393eedfed4a87dab0ce6c8f705af168eeffef3ef2a9a6a4483e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:38:07.091 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:07 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:38:07.092 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:07 smithi026 systemd[1]: Stopped Ceph osd.3 for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:07.092 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 13 14:38:07 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.3.service: Consumed 50.067s CPU time. 2024-09-13T14:38:07.107 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:07.107 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T14:38:07.107 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T14:38:07.108 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.4 2024-09-13T14:38:07.138 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:06.940+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:07.139 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:06.944+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:07.139 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:06 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:06.885+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:07.405 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:07 smithi145 systemd[1]: Stopping Ceph osd.4 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:07.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:07.404+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:07.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:07.409+0000 7f1d814be640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:07.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:07.409+0000 7f1d814be640 -1 osd.4 1034 *** Got signal Terminated *** 2024-09-13T14:38:07.706 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:07.409+0000 7f1d814be640 -1 osd.4 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:08.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:07.883+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:08.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:07.953+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:08.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:07 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:07.924+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:08.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:08 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:08.411+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:09.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:08 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:08.858+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:09.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:08 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:08.974+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:09.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:08 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:08.937+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:09.827 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:09.461+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:10.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:09.826+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:10.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:09.981+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:10.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:09.948+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:10.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:09 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:09.948+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:10.872 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:10.486+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:11.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:10.871+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:11.205 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:10.902+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:11.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:10 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:10.902+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:11.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:11.028+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:11.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:11.028+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:11.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:11.532+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:11.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:11.532+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:12.205 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4[95229]: 2024-09-13T14:38:11.882+0000 7f1d79acb640 -1 osd.4 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:39.849046+0000 front 2024-09-13T14:37:39.848874+0000 (oldest deadline 2024-09-13T14:38:02.748791+0000) 2024-09-13T14:38:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:11.990+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:12.206 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:11.990+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:12.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:11.905+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:12.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:11 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:11.905+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:12.910 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:12.512+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:12.910 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:12.512+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:13.184 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:13.029+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:13.184 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:13.029+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:13.185 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:12 smithi145 podman[124333]: 2024-09-13 14:38:12.922376807 +0000 UTC m=+5.613485872 container died 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T14:38:13.185 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:12.909+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:13.185 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:12 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:12.909+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:13.451 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:13 smithi145 podman[124333]: 2024-09-13 14:38:13.266602606 +0000 UTC m=+5.957711655 container cleanup 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:38:13.452 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:13 smithi145 bash[124333]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T14:38:13.705 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:13 smithi145 podman[124365]: 2024-09-13 14:38:13.451152893 +0000 UTC m=+0.527413046 container remove 1d10fdf8c82ad7cf7f949bd760a7a4a3a46f04dfde5773a0f5e2172303df6a84 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:38:13.706 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:13.471+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:13.706 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:13.471+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:14.130 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:14.039+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:14.130 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:14.039+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:14.130 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:13 smithi145 podman[124460]: 2024-09-13 14:38:13.781906881 +0000 UTC m=+0.015611964 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:14.130 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:13 smithi145 podman[124460]: 2024-09-13 14:38:13.902169299 +0000 UTC m=+0.135874417 container create f82a68d1a95b5e2bc9e0899a504a7bbc7b43be0557eb13f76f673c02624b0343 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:38:14.131 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:13.909+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:14.131 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:13 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:13.909+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:14.455 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:14 smithi145 podman[124460]: 2024-09-13 14:38:14.129622131 +0000 UTC m=+0.363327215 container init f82a68d1a95b5e2bc9e0899a504a7bbc7b43be0557eb13f76f673c02624b0343 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:14.455 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:14 smithi145 podman[124460]: 2024-09-13 14:38:14.134353238 +0000 UTC m=+0.368058318 container start f82a68d1a95b5e2bc9e0899a504a7bbc7b43be0557eb13f76f673c02624b0343 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:38:14.455 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:14 smithi145 podman[124460]: 2024-09-13 14:38:14.196148809 +0000 UTC m=+0.429853886 container attach f82a68d1a95b5e2bc9e0899a504a7bbc7b43be0557eb13f76f673c02624b0343 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:38:14.455 INFO:journalctl@ceph.osd.4.smithi145.stdout:Sep 13 14:38:14 smithi145 podman[124460]: 2024-09-13 14:38:14.332414075 +0000 UTC m=+0.566119158 container died f82a68d1a95b5e2bc9e0899a504a7bbc7b43be0557eb13f76f673c02624b0343 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:14.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:14.453+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:14.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:14.453+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:14.934 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T14:38:15.015 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:15.016 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T14:38:15.016 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T14:38:15.016 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.5 2024-09-13T14:38:15.205 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 systemd[1]: Stopping Ceph osd.5 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:15.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:14.992+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:15.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:14.992+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:15.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:14.952+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:15.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:14.952+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.921Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.206 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.921Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.923Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.924Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.927Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.207 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:14 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:14.927Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-09-13T14:38:15.708 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:15.354+0000 7f359e5a1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:15.708 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:15.354+0000 7f359e5a1640 -1 osd.5 1034 *** Got signal Terminated *** 2024-09-13T14:38:15.709 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:15.354+0000 7f359e5a1640 -1 osd.5 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:15.709 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:15.417+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:15.709 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:15.417+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:16.423 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:16.016+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:16.423 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:16.016+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:16.423 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:15.963+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:16.423 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:15 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:15.963+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:16.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:16.422+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:16.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:16 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:16.422+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:17.415 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:16.993Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-09-13T14:38:17.466 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:16 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:16.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-09-13T14:38:17.467 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:17.041+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:17.467 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:17.041+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:17.468 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:17.005+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:17.468 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:17.005+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:17.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:17.453+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:17.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:17.453+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:18.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:18.034+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:18.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:18.034+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:18.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:17.973+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:18.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:17 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:17.973+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:18.955 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:18.458+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:18.955 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:18.458+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:19.440 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:19.028+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:19.440 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:19.028+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:19.440 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:18.997+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:19.440 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:18.997+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:19.441 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:18 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:18.997+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:19.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:19.439+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.476095+0000 front 2024-09-13T14:37:40.475855+0000 (oldest deadline 2024-09-13T14:38:03.976042+0000) 2024-09-13T14:38:19.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5[100152]: 2024-09-13T14:38:19.439+0000 7f359abb6640 -1 osd.5 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:48.577609+0000 front 2024-09-13T14:37:48.577573+0000 (oldest deadline 2024-09-13T14:38:10.877413+0000) 2024-09-13T14:38:20.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:20.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:20.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:20.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:20.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:20.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:20.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:19.952+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:20.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:19.952+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:20.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:19 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:19.952+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:21.029 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:20 smithi145 podman[124591]: 2024-09-13 14:38:20.750110175 +0000 UTC m=+5.504521906 container died b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T14:38:21.029 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:20.911+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:21.030 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:20.911+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:21.030 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:20 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:20.911+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:21.387 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124591]: 2024-09-13 14:38:21.127638564 +0000 UTC m=+5.882050293 container cleanup b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T14:38:21.387 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 bash[124591]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T14:38:21.387 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:21.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:21.388 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:21.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:21.388 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:21.067+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:21.705 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124612]: 2024-09-13 14:38:21.387041065 +0000 UTC m=+0.633609802 container remove b9422108344c651ba98b492f7d48ba326592e2e316b165f911c4e2637b39bf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T14:38:22.058 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:21.954+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:22.058 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:21.954+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:22.059 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:21 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:21.954+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:22.059 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124709]: 2024-09-13 14:38:21.660073954 +0000 UTC m=+0.015994318 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:22.059 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124709]: 2024-09-13 14:38:21.779825158 +0000 UTC m=+0.135745509 container create d8ddeb4912348bf2eaedbdd52d6aad1a2273df27b7958aaf98a120a95603b4d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:38:22.059 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124709]: 2024-09-13 14:38:21.982687649 +0000 UTC m=+0.338608002 container init d8ddeb4912348bf2eaedbdd52d6aad1a2273df27b7958aaf98a120a95603b4d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T14:38:22.059 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:21 smithi145 podman[124709]: 2024-09-13 14:38:21.987964352 +0000 UTC m=+0.343884705 container start d8ddeb4912348bf2eaedbdd52d6aad1a2273df27b7958aaf98a120a95603b4d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T14:38:22.455 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:22 smithi145 podman[124709]: 2024-09-13 14:38:22.083053193 +0000 UTC m=+0.438973546 container attach d8ddeb4912348bf2eaedbdd52d6aad1a2273df27b7958aaf98a120a95603b4d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:38:22.456 INFO:journalctl@ceph.osd.5.smithi145.stdout:Sep 13 14:38:22 smithi145 podman[124709]: 2024-09-13 14:38:22.172785824 +0000 UTC m=+0.528706187 container died d8ddeb4912348bf2eaedbdd52d6aad1a2273df27b7958aaf98a120a95603b4d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T14:38:22.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:22.057+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:22.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:22.057+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:22.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:22.057+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:22.751 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T14:38:22.788 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:22.789 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T14:38:22.789 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T14:38:22.789 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.6 2024-09-13T14:38:23.025 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:22 smithi145 systemd[1]: Stopping Ceph osd.6 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:23.025 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:22.970+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:23.025 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:22.970+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:23.026 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:22 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:22.970+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:23.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:23.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:23.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:23.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.140+0000 7f76834bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:23.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.140+0000 7f76834bd640 -1 osd.6 1034 *** Got signal Terminated *** 2024-09-13T14:38:23.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:23.140+0000 7f76834bd640 -1 osd.6 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:24.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:24.069+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:24.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:24.069+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:24.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:24 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:24.069+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:24.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:23.982+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:24.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:23.982+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:24.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:23 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:23.982+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:25.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:25.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:25.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:25.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:25.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:25.024+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:25.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:25.014+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:25.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:25.014+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:25.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:25 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:25.014+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:26.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:26.007+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:26.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:26.007+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:26.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:26.007+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:26.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:26.066+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:26.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:26.066+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:26.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:26 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:26.066+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:27.327 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:26.993Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-09-13T14:38:27.327 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:26.993Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-09-13T14:38:27.327 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:26 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:26.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-09-13T14:38:27.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:27.021+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:27.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:27.021+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:27.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:27.021+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:27.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:27.038+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:27.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:27.038+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:27.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:27 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:27.038+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:28.455 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:28.063+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.266428+0000 front 2024-09-13T14:37:40.266464+0000 (oldest deadline 2024-09-13T14:38:01.366256+0000) 2024-09-13T14:38:28.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:28.063+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:46.567575+0000 front 2024-09-13T14:37:46.567671+0000 (oldest deadline 2024-09-13T14:38:10.067496+0000) 2024-09-13T14:38:28.456 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6[105330]: 2024-09-13T14:38:28.063+0000 7f767fad2640 -1 osd.6 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.368668+0000 front 2024-09-13T14:37:55.368622+0000 (oldest deadline 2024-09-13T14:38:19.468594+0000) 2024-09-13T14:38:28.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:28.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:28.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:28.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:28.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:28 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:28.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:28.949 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:28 smithi145 podman[124838]: 2024-09-13 14:38:28.676689417 +0000 UTC m=+5.664423557 container died 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:38:29.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124838]: 2024-09-13 14:38:29.046951369 +0000 UTC m=+6.034685507 container cleanup 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:38:29.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 bash[124838]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6 2024-09-13T14:38:29.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:29 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:29.054+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:29.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:29 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:29.054+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:29.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:29 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:29.054+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:29.612 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124859]: 2024-09-13 14:38:29.298111744 +0000 UTC m=+0.617780558 container remove 304ffd9a14d808e8fae8f5b1bc0b182f554f114ef2f15a7e0032c82d00099378 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:38:29.865 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124954]: 2024-09-13 14:38:29.514630295 +0000 UTC m=+0.015706878 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:29.865 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124954]: 2024-09-13 14:38:29.615598181 +0000 UTC m=+0.116674801 container create dab23cf0f0a02a21a39f6422b295c8939efbce57f307f45c5ceb50f4a18c8536 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True) 2024-09-13T14:38:29.865 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124954]: 2024-09-13 14:38:29.860046911 +0000 UTC m=+0.361123489 container init dab23cf0f0a02a21a39f6422b295c8939efbce57f307f45c5ceb50f4a18c8536 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:29.865 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124954]: 2024-09-13 14:38:29.865016098 +0000 UTC m=+0.366092677 container start dab23cf0f0a02a21a39f6422b295c8939efbce57f307f45c5ceb50f4a18c8536 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:38:30.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:29 smithi145 podman[124954]: 2024-09-13 14:38:29.968220232 +0000 UTC m=+0.469296811 container attach dab23cf0f0a02a21a39f6422b295c8939efbce57f307f45c5ceb50f4a18c8536 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T14:38:30.205 INFO:journalctl@ceph.osd.6.smithi145.stdout:Sep 13 14:38:30 smithi145 podman[124954]: 2024-09-13 14:38:30.058185816 +0000 UTC m=+0.559262396 container died dab23cf0f0a02a21a39f6422b295c8939efbce57f307f45c5ceb50f4a18c8536 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T14:38:30.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:30.096+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:30.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:30.096+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:30.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:30.096+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:30.206 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:30 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:30.096+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:30.638 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T14:38:30.675 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:30.676 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T14:38:30.676 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T14:38:30.676 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.7 2024-09-13T14:38:30.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:30 smithi145 systemd[1]: Stopping Ceph osd.7 for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:31.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.000+0000 7fc7a9248640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:31.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.000+0000 7fc7a9248640 -1 osd.7 1034 *** Got signal Terminated *** 2024-09-13T14:38:31.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.000+0000 7fc7a9248640 -1 osd.7 1034 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:31.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.087+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:31.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.087+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:31.457 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.087+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:31.457 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:31 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:31.087+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:32.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:32.039+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:32.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:32.039+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:32.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:32.039+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:32.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:32 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:32.039+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:33.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:33.065+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:33.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:33.065+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:33.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:33.065+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:33.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:33 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:33.065+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:34.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:34.044+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:34.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:34.044+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:34.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:34.044+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:34.456 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:34 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:34.044+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:35.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:35.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:35.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:35.455 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.029+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:36.362 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.990+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-09-13T14:37:40.191135+0000 front 2024-09-13T14:37:40.191052+0000 (oldest deadline 2024-09-13T14:38:03.690830+0000) 2024-09-13T14:38:36.362 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.990+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-09-13T14:37:43.691424+0000 front 2024-09-13T14:37:43.691346+0000 (oldest deadline 2024-09-13T14:38:09.591168+0000) 2024-09-13T14:38:36.362 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.990+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-09-13T14:37:55.992709+0000 front 2024-09-13T14:37:55.992598+0000 (oldest deadline 2024-09-13T14:38:18.892499+0000) 2024-09-13T14:38:36.362 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.990+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-09-13T14:38:04.793449+0000 front 2024-09-13T14:38:04.793414+0000 (oldest deadline 2024-09-13T14:38:29.493283+0000) 2024-09-13T14:38:36.362 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:35 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7[110474]: 2024-09-13T14:38:35.990+0000 7fc7a585d640 -1 osd.7 1034 heartbeat_check: no reply from 172.21.15.145:6804 osd.4 since back 2024-09-13T14:38:09.493803+0000 front 2024-09-13T14:38:09.493825+0000 (oldest deadline 2024-09-13T14:38:35.393724+0000) 2024-09-13T14:38:36.643 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:36 smithi145 podman[125083]: 2024-09-13 14:38:36.361656004 +0000 UTC m=+5.466395160 container died 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:38:36.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:36 smithi145 podman[125083]: 2024-09-13 14:38:36.749457571 +0000 UTC m=+5.854196769 container cleanup 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T14:38:36.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:36 smithi145 bash[125083]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7 2024-09-13T14:38:37.304 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:36 smithi145 podman[125104]: 2024-09-13 14:38:36.983898069 +0000 UTC m=+0.619133398 container remove 606a2c6e152f52ffda2c4fd8df06a066ee48ec46cfb5d32d93128ec902203834 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T14:38:37.415 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:36 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:36.994Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-09-13T14:38:37.571 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.206458833 +0000 UTC m=+0.016045170 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:38:37.571 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.309661423 +0000 UTC m=+0.119247802 container create 9098d41b8ee6b7f632ef2af0c07fd286dad8d777ea7a49f4fdcca010fe5841ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T14:38:37.571 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.495722518 +0000 UTC m=+0.305308855 container init 9098d41b8ee6b7f632ef2af0c07fd286dad8d777ea7a49f4fdcca010fe5841ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T14:38:37.571 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.500511008 +0000 UTC m=+0.310097345 container start 9098d41b8ee6b7f632ef2af0c07fd286dad8d777ea7a49f4fdcca010fe5841ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T14:38:37.571 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.570551304 +0000 UTC m=+0.380137653 container attach 9098d41b8ee6b7f632ef2af0c07fd286dad8d777ea7a49f4fdcca010fe5841ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:38:37.955 INFO:journalctl@ceph.osd.7.smithi145.stdout:Sep 13 14:38:37 smithi145 podman[125198]: 2024-09-13 14:38:37.686392514 +0000 UTC m=+0.495978864 container died 9098d41b8ee6b7f632ef2af0c07fd286dad8d777ea7a49f4fdcca010fe5841ce (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True) 2024-09-13T14:38:38.389 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T14:38:38.433 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:38.433 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T14:38:38.433 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-13T14:38:38.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.a 2024-09-13T14:38:38.756 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:38 smithi026 systemd[1]: Stopping Ceph mds.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:39.105 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: 2024-09-13T14:38:38.754+0000 7f6081dfb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:39.105 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:38 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a[125507]: 2024-09-13T14:38:38.754+0000 7f6081dfb640 -1 mds.a *** got signal Terminated *** 2024-09-13T14:38:43.054 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:42 smithi026 podman[136609]: 2024-09-13 14:38:42.764721635 +0000 UTC m=+4.132991308 container died e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T14:38:43.386 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:43 smithi026 podman[136609]: 2024-09-13 14:38:43.127170432 +0000 UTC m=+4.495440104 container cleanup e163e8cfc713b99772d5bf8baa66de527b70abd9f26fbae32a4cd2001742a919 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:38:43.387 INFO:journalctl@ceph.mds.a.smithi026.stdout:Sep 13 14:38:43 smithi026 bash[136609]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-a 2024-09-13T14:38:43.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.a.service' 2024-09-13T14:38:43.607 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:43.607 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-13T14:38:43.607 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-13T14:38:43.608 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.b 2024-09-13T14:38:43.955 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:43 smithi145 systemd[1]: Stopping Ceph mds.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:44.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[115263]: 2024-09-13T14:38:43.969+0000 7f3b51c6a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:44.455 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:43 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b[115263]: 2024-09-13T14:38:43.969+0000 7f3b51c6a640 -1 mds.b *** got signal Terminated *** 2024-09-13T14:38:47.694 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:47 smithi145 podman[125326]: 2024-09-13 14:38:47.377538023 +0000 UTC m=+3.575326231 container died 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:38:47.955 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:47 smithi145 podman[125326]: 2024-09-13 14:38:47.78410363 +0000 UTC m=+3.981891840 container cleanup 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T14:38:47.955 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:47 smithi145 bash[125326]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b 2024-09-13T14:38:47.955 INFO:journalctl@ceph.mds.b.smithi145.stdout:Sep 13 14:38:47 smithi145 podman[125355]: 2024-09-13 14:38:47.935252459 +0000 UTC m=+0.549223898 container remove 859c720bf4267ec5e20f035552f47aa796e98f6fdc493ae0e4fc7540dd842a12 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-mds-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-13T14:38:48.159 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@mds.b.service' 2024-09-13T14:38:48.195 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:48.196 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-13T14:38:48.196 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T14:38:48.196 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T14:38:48.455 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 systemd[1]: Stopping Ceph prometheus.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:48.822 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:38:48.823 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T14:38:48.824 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.549Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T14:38:48.824 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.550Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:38:48.824 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.550Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T14:38:48.824 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a[80013]: ts=2024-09-13T14:38:48.550Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T14:38:48.824 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 podman[125471]: 2024-09-13 14:38:48.559822829 +0000 UTC m=+0.145603623 container died 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:49.197 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 podman[125471]: 2024-09-13 14:38:48.921245732 +0000 UTC m=+0.507026517 container cleanup 106373b0ae43e217f69b46a4d6bb4a83354eb8eac46ae0e25437793eebf923a6 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:49.198 INFO:journalctl@ceph.prometheus.a.smithi145.stdout:Sep 13 14:38:48 smithi145 bash[125471]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:38:49.343 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad455616-71d9-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T14:38:49.376 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:38:49.376 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T14:38:49.377 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad455616-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T14:38:51.376 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:51 smithi026 systemd[1]: Stopping Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:51.376 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:51 smithi026 ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a[87052]: ts=2024-09-13T14:38:51.334Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:38:51.455 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore.transactions t=2024-09-13T14:38:51.182714602Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:38:51.456 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=cleanup t=2024-09-13T14:38:51.187402967Z level=info msg="Completed cleanup jobs" duration=15.082659ms 2024-09-13T14:38:51.456 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=plugins.update.checker t=2024-09-13T14:38:51.312030909Z level=info msg="Update check succeeded" duration=57.86625ms 2024-09-13T14:38:51.456 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:51 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafana.update.checker t=2024-09-13T14:38:51.32337878Z level=info msg="Update check succeeded" duration=68.104766ms 2024-09-13T14:38:51.658 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:51 smithi026 podman[137058]: 2024-09-13 14:38:51.375526202 +0000 UTC m=+0.137948095 container died 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:51.915 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:51 smithi026 podman[137058]: 2024-09-13 14:38:51.757398307 +0000 UTC m=+0.519820199 container cleanup 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:51.916 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:51 smithi026 bash[137058]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:38:52.400 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:52 smithi026 podman[137076]: 2024-09-13 14:38:52.000173445 +0000 UTC m=+0.618089667 container remove 4b02ef02803fddfaf2424ba51b46a9051ffeb33b3943e5792f65b4e10da1b391 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:52.400 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:52 smithi026 podman[137076]: 2024-09-13 14:38:52.084084596 +0000 UTC m=+0.702000746 volume remove 8778078b5afbe3bdb7db8aae304013a3e25a6e7f2b7cf7538dfe2fb189edcdef 2024-09-13T14:38:52.400 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:52 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:38:52.400 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:52 smithi026 systemd[1]: Stopped Ceph alertmanager.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:52.401 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 13 14:38:52 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.906s CPU time. 2024-09-13T14:38:52.665 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:52 smithi026 systemd[1]: Stopping Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:52.938 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:52 smithi026 podman[137190]: 2024-09-13 14:38:52.666123567 +0000 UTC m=+0.123822607 container died 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:53.354 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 podman[137190]: 2024-09-13 14:38:53.162027506 +0000 UTC m=+0.619726552 container cleanup 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:53.355 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 bash[137190]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:38:53.640 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 podman[137208]: 2024-09-13 14:38:53.354505211 +0000 UTC m=+0.691870515 container remove 21fa336e551b20b0f45479a5221e51bcbde16c1f985728076d359dd18c501bcc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:38:53.640 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:38:53.640 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:38:53.640 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 systemd[1]: Stopped Ceph node-exporter.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:53.640 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 13 14:38:53 smithi026 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.859s CPU time. 2024-09-13T14:38:54.133 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad455616-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T14:38:56.104 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:55 smithi145 systemd[1]: Stopping Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=server t=2024-09-13T14:38:56.103635266Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=tracing t=2024-09-13T14:38:56.103697935Z level=info msg="Closing tracing" 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=ticker t=2024-09-13T14:38:56.103715034Z level=info msg=stopped last_tick=2024-09-13T14:38:50Z 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=grafana-apiserver t=2024-09-13T14:38:56.103760075Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a[119489]: logger=sqlstore.transactions t=2024-09-13T14:38:56.114755636Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:38:56.361 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 podman[125922]: 2024-09-13 14:38:56.124414836 +0000 UTC m=+0.136257656 container died 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:38:56.657 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 podman[125922]: 2024-09-13 14:38:56.443071126 +0000 UTC m=+0.454913935 container cleanup 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:38:56.657 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 bash[125922]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:38:56.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 podman[125940]: 2024-09-13 14:38:56.656701996 +0000 UTC m=+0.532559682 container remove 4b007f928313350ca8d3badd271e8a6816ffedc7801be9052058660b2f68af23 (image=quay.io/ceph/grafana:10.4.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:38:56.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T14:38:56.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 systemd[1]: Stopped Ceph grafana.a for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:56.955 INFO:journalctl@ceph.grafana.a.smithi145.stdout:Sep 13 14:38:56 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@grafana.a.service: Consumed 6.886s CPU time. 2024-09-13T14:38:57.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 systemd[1]: Stopping Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:57.455 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 podman[126051]: 2024-09-13 14:38:57.293178167 +0000 UTC m=+0.109185649 container died a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:38:57.915 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 podman[126051]: 2024-09-13 14:38:57.638413251 +0000 UTC m=+0.454420730 container cleanup a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:38:57.915 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 bash[126051]: ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:38:58.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 podman[126069]: 2024-09-13 14:38:57.914580539 +0000 UTC m=+0.625433462 container remove a22f428718b1ed0dfd0f4eef3651e08538fb62aaf92bfcc24148bf018126765d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ad455616-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:38:58.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:57 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:38:58.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:58 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:38:58.205 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:58 smithi145 systemd[1]: Stopped Ceph node-exporter.b for ad455616-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:58.206 INFO:journalctl@ceph.node-exporter.b.smithi145.stdout:Sep 13 14:38:58 smithi145 systemd[1]: ceph-ad455616-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 3.614s CPU time. 2024-09-13T14:38:58.708 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:38:58.742 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:38:58.770 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T14:38:58.858 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi026/crash 2024-09-13T14:38:58.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T14:38:58.908 INFO:teuthology.orchestra.run.smithi026.stderr:tar: /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T14:38:58.909 INFO:teuthology.orchestra.run.smithi026.stderr:tar: Error is not recoverable: exiting now 2024-09-13T14:38:58.911 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi145/crash 2024-09-13T14:38:58.911 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T14:38:58.945 INFO:teuthology.orchestra.run.smithi145.stderr:tar: /var/lib/ceph/ad455616-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T14:38:58.946 INFO:teuthology.orchestra.run.smithi145.stderr:tar: Error is not recoverable: exiting now 2024-09-13T14:38:58.947 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T14:38:58.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-13T14:38:58.987 INFO:teuthology.orchestra.run.smithi026.stdout:2024-09-13T14:27:42.966417+0000 mon.a (mon.0) 822 : cluster [WRN] Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T14:38:58.988 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-13T14:38:58.988 INFO:tasks.cephadm:Compressing logs... 2024-09-13T14:38:58.988 DEBUG:teuthology.orchestra.run.smithi026:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T14:38:59.030 DEBUG:teuthology.orchestra.run.smithi145:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T14:38:59.054 INFO:teuthology.orchestra.run.smithi145.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T14:38:59.054 INFO:teuthology.orchestra.run.smithi145.stderr:: No such file or directory 2024-09-13T14:38:59.054 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T14:38:59.062 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.b.log 2024-09-13T14:38:59.062 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log: /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T14:38:59.062 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T14:38:59.062 INFO:teuthology.orchestra.run.smithi145.stderr: 84.4% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.b.log.gz 2024-09-13T14:38:59.063 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose --/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T14:38:59.063 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T14:38:59.063 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T14:38:59.064 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T14:38:59.064 INFO:teuthology.orchestra.run.smithi145.stderr: 84.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T14:38:59.064 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T14:38:59.064 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T14:38:59.064 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log: /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log: 90.4% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T14:38:59.067 INFO:teuthology.orchestra.run.smithi026.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T14:38:59.067 INFO:teuthology.orchestra.run.smithi026.stderr:: No such file or directory 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.a.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:gzip/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log: -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T14:38:59.068 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: 90.9% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log: 91.8% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mds.a.log.gz 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr: 94.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T14:38:59.069 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T14:38:59.070 INFO:teuthology.orchestra.run.smithi145.stderr: 85.9% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T14:38:59.071 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T14:38:59.077 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log: 91.2% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T14:38:59.077 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T14:38:59.080 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T14:38:59.089 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log: 92.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T14:38:59.089 INFO:teuthology.orchestra.run.smithi145.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T14:38:59.092 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T14:38:59.120 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T14:38:59.127 INFO:teuthology.orchestra.run.smithi145.stderr: 89.8% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T14:38:59.530 INFO:teuthology.orchestra.run.smithi026.stderr: 89.2% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T14:38:59.615 INFO:teuthology.orchestra.run.smithi145.stderr: 92.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T14:38:59.830 INFO:teuthology.orchestra.run.smithi026.stderr: 92.0% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T14:39:00.963 INFO:teuthology.orchestra.run.smithi026.stderr: 90.9% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T14:39:03.466 INFO:teuthology.orchestra.run.smithi145.stderr: 94.3% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T14:39:04.208 INFO:teuthology.orchestra.run.smithi145.stderr: 94.4% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T14:39:04.524 INFO:teuthology.orchestra.run.smithi145.stderr: 94.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T14:39:04.707 INFO:teuthology.orchestra.run.smithi145.stderr: 94.1% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T14:39:04.709 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-09-13T14:39:04.709 INFO:teuthology.orchestra.run.smithi145.stderr:real 0m5.665s 2024-09-13T14:39:04.710 INFO:teuthology.orchestra.run.smithi145.stderr:user 0m20.518s 2024-09-13T14:39:04.710 INFO:teuthology.orchestra.run.smithi145.stderr:sys 0m0.833s 2024-09-13T14:39:05.739 INFO:teuthology.orchestra.run.smithi026.stderr: 93.9% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T14:39:06.038 INFO:teuthology.orchestra.run.smithi026.stderr: 93.8% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T14:39:07.495 INFO:teuthology.orchestra.run.smithi026.stderr: 94.5% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T14:39:08.126 INFO:teuthology.orchestra.run.smithi026.stderr: 94.3% -- replaced with /var/log/ceph/ad455616-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T14:39:08.128 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-09-13T14:39:08.128 INFO:teuthology.orchestra.run.smithi026.stderr:real 0m9.084s 2024-09-13T14:39:08.128 INFO:teuthology.orchestra.run.smithi026.stderr:user 0m33.147s 2024-09-13T14:39:08.128 INFO:teuthology.orchestra.run.smithi026.stderr:sys 0m1.125s 2024-09-13T14:39:08.128 INFO:tasks.cephadm:Archiving logs... 2024-09-13T14:39:08.129 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi026/log 2024-09-13T14:39:08.130 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T14:39:12.433 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi145/log 2024-09-13T14:39:12.434 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T14:39:15.388 INFO:tasks.cephadm:Removing cluster... 2024-09-13T14:39:15.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad455616-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T14:39:15.917 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad455616-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T14:39:16.441 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T14:39:16.441 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T14:39:16.458 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T14:39:16.473 INFO:tasks.cephadm:Teardown complete 2024-09-13T14:39:16.473 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T14:39:16.484 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T14:39:16.485 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T14:39:16.502 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T14:39:16.606 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T14:39:16.616 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T14:39:16.617 DEBUG:teuthology.orchestra.run.smithi026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:39:16.619 DEBUG:teuthology.orchestra.run.smithi145:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:39:16.633 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:39:16.637 INFO:teuthology.orchestra.run.smithi145.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:39:16.637 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:39:16.638 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-09-13T14:39:16.638 INFO:teuthology.orchestra.run.smithi026.stdout:^* hv01.front.sepia.ceph.com 3 6 377 45 +255us[ +252us] +/- 61ms 2024-09-13T14:39:16.638 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 49 -843us[ -846us] +/- 61ms 2024-09-13T14:39:16.638 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 45 +775us[ +772us] +/- 76ms 2024-09-13T14:39:16.638 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:39:16.639 INFO:teuthology.orchestra.run.smithi145.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:39:16.639 INFO:teuthology.orchestra.run.smithi145.stdout:=============================================================================== 2024-09-13T14:39:16.639 INFO:teuthology.orchestra.run.smithi145.stdout:^* hv01.front.sepia.ceph.com 3 7 377 50 +261us[ +290us] +/- 61ms 2024-09-13T14:39:16.640 INFO:teuthology.orchestra.run.smithi145.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 49 -871us[ -871us] +/- 61ms 2024-09-13T14:39:16.640 INFO:teuthology.orchestra.run.smithi145.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 44 +770us[ +770us] +/- 76ms 2024-09-13T14:39:16.640 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:39:16.640 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T14:39:16.652 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T14:39:16.653 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T14:39:16.684 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T14:39:16.778 DEBUG:teuthology.orchestra.run.smithi145:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T14:39:16.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:39:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.890 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:39:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.919 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T14:39:16.931 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T14:39:16.952 INFO:teuthology.task.internal:Duration was 2158.738427 seconds 2024-09-13T14:39:16.953 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T14:39:16.986 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T14:39:16.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T14:39:16.989 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:39:17.239 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T14:39:17.239 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com 2024-09-13T14:39:17.239 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T14:39:17.267 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi145.front.sepia.ceph.com 2024-09-13T14:39:17.267 DEBUG:teuthology.orchestra.run.smithi145:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T14:39:17.306 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T14:39:17.306 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T14:39:17.309 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T14:39:17.373 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T14:39:17.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T14:39:17.376 DEBUG:teuthology.orchestra.run.smithi145:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T14:39:17.829 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T14:39:17.840 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T14:39:17.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T14:39:17.869 DEBUG:teuthology.orchestra.run.smithi145:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T14:39:17.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T14:39:17.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T14:39:17.912 DEBUG:teuthology.orchestra.run.smithi145:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T14:39:17.934 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core 2024-09-13T14:39:17.958 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = core 2024-09-13T14:39:17.980 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T14:39:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.003 DEBUG:teuthology.orchestra.run.smithi145:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T14:39:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.035 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T14:39:18.046 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T14:39:18.047 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi026 2024-09-13T14:39:18.048 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T14:39:18.097 DEBUG:teuthology.misc:Transferring archived files from smithi145:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903551/remote/smithi145 2024-09-13T14:39:18.098 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T14:39:18.142 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T14:39:18.143 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T14:39:18.145 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T14:39:18.192 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T14:39:18.206 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T14:39:18.206 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T14:39:18.229 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T14:39:18.229 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T14:39:18.232 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T14:39:18.245 INFO:teuthology.orchestra.run.smithi026.stdout: 265701 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 14:39 /home/ubuntu/cephtest 2024-09-13T14:39:18.245 INFO:teuthology.orchestra.run.smithi026.stdout: 548894 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 14:32 /home/ubuntu/cephtest/mnt.0 2024-09-13T14:39:18.245 INFO:teuthology.orchestra.run.smithi026.stdout: 548895 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 14:36 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:39:18.245 INFO:teuthology.orchestra.run.smithi026.stdout: 660215 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 14:36 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:39:18.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.247 INFO:teuthology.orchestra.run.smithi026.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T14:39:18.247 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T14:39:18.254 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T14:39:18.267 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T14:39:18.296 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T14:39:18.302 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 2158.738427400589 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi026 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_akupczyk@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=5b34c29ea8d54cee9bb0cafdc5cdc28c status: fail success: false 2024-09-13T14:39:18.302 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T14:39:18.332 INFO:teuthology.orchestra.run.smithi145.stdout: 266054 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 14:39 /home/ubuntu/cephtest 2024-09-13T14:39:18.759 INFO:teuthology.run:FAIL